{ "metadata": { "benchmark": "Tax Agent Bench", "slug": "tax_agent_bench", "description": "Evaluating agents on research-grade US tax questions using the Tax Agent Bench harness", "benchmark_id": "tax_agent_bench", "family": "tax_agent_bench", "version": "1", "updated": "2026-10-18", "dataset_type": "private", "industry": "finance", "tasks": { "overall": "Overall", "weighted_pass": "Weighted Pass", "fact_pattern_analysis": "Fact-Pattern Analysis", "rule_source_lookup": "Rule & Source Lookup", "current_temporal_analysis": "Current & Temporal Analysis", "numbers_calculations": "Numbers & Calculations", "forms_filings": "Forms & Filings", "controversy_precedence_analysis": "Controversy & Precedence Analysis", "weighted_pass__fact_pattern_analysis": "Fact-Pattern Analysis", "weighted_pass__rule_source_lookup": "Rule & Source Lookup", "weighted_pass__current_temporal_analysis": "Current & Temporal Analysis", "weighted_pass__numbers_calculations": "Numbers & Calculations", "weighted_pass__forms_filings": "Forms & Filings", "weighted_pass__controversy_precedence_analysis": "Controversy & Precedence Analysis" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "fireworks/ember-1", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-large-4", "mistralai/mistral-medium-3.5", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 71 }, "tasks": { "overall": { "anthropic/claude-fable-5-1": { "accuracy": 49.194, "latency": 1714.017, "stderr": 3.606, "cost_per_test": 13.180073, "token_totals": { "input_tokens": 449166918, "output_tokens": 28237610, "reasoning_tokens": 7224, "cache_read_tokens": 365930262, "cache_write_tokens": 83209817 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 45.573, "latency": 4012.548, "stderr": 3.593, "cost_per_test": 15.086184, "token_totals": { "input_tokens": 815304119, "output_tokens": 88000495, "reasoning_tokens": 76703031, "cache_read_tokens": 648981098, "cache_write_tokens": 166280701 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 45.548, "latency": 1008.306, "stderr": 3.591, "cost_per_test": 5.136554, "token_totals": { "input_tokens": 322077193, "output_tokens": 16761490, "reasoning_tokens": 45865, "cache_read_tokens": 250544336, "cache_write_tokens": 71504899 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 44.56, "latency": 985.634, "stderr": 3.587, "cost_per_test": 3.473645, "token_totals": { "input_tokens": 285519334, "output_tokens": 14493886, "reasoning_tokens": 6146392, "cache_read_tokens": 200405700, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 44.35, "latency": 1264.434, "stderr": 3.574, "cost_per_test": 0.459454, "token_totals": { "input_tokens": 779160448, "output_tokens": 53561725, "reasoning_tokens": 45716077, "cache_read_tokens": 712066688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 42.997, "latency": 1084.296, "stderr": 3.572, "cost_per_test": 2.147235, "token_totals": { "input_tokens": 257606995, "output_tokens": 9968530, "reasoning_tokens": 6086406, "cache_read_tokens": 214574250, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 42.435, "latency": 248.818, "stderr": 3.511, "cost_per_test": 0.389917, "token_totals": { "input_tokens": 142091226, "output_tokens": 6095824, "reasoning_tokens": 3452149, "cache_read_tokens": 116606720, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 41.959, "latency": 3560.542, "stderr": 3.561, "cost_per_test": 9.245466, "token_totals": { "input_tokens": 1150865417, "output_tokens": 108029684, "reasoning_tokens": 94199819, "cache_read_tokens": 954147139, "cache_write_tokens": 196666034 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 39.698, "latency": 3052.932, "stderr": 3.52, "cost_per_test": 1.80097, "token_totals": { "input_tokens": 229302538, "output_tokens": 47417212, "reasoning_tokens": 1131, "cache_read_tokens": 154199490, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 38.281, "latency": 866.628, "stderr": 3.503, "cost_per_test": 6.696087, "token_totals": { "input_tokens": 104187949, "output_tokens": 14106474, "reasoning_tokens": 10318710, "cache_read_tokens": 62150339, "cache_write_tokens": 42027339 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 36.269, "latency": 492.843, "stderr": 3.47, "cost_per_test": 1.884439, "token_totals": { "input_tokens": 225535207, "output_tokens": 8770631, "reasoning_tokens": 5466021, "cache_read_tokens": 187857044, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 35.974, "latency": 505.365, "stderr": 3.42, "cost_per_test": 0.319558, "token_totals": { "input_tokens": 95475291, "output_tokens": 4737687, "reasoning_tokens": 2609786, "cache_read_tokens": 70731373, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 34.197, "latency": 996.469, "stderr": 3.423, "cost_per_test": 1.796539, "token_totals": { "input_tokens": 280409764, "output_tokens": 10821528, "reasoning_tokens": 6708987, "cache_read_tokens": 186011136, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 33.368, "latency": 2415.159, "stderr": 3.386, "cost_per_test": 2.790127, "token_totals": { "input_tokens": 444117224, "output_tokens": 14995598, "reasoning_tokens": 10717157, "cache_read_tokens": 381463574, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.6": { "accuracy": 33.141, "latency": 657.844, "stderr": 3.396, "cost_per_test": 0.978328, "token_totals": { "input_tokens": 139046016, "output_tokens": 7673212, "reasoning_tokens": 4482527, "cache_read_tokens": 90209280, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 32.798, "latency": 119.905, "stderr": 2.751, "cost_per_test": 0.269212, "token_totals": { "input_tokens": 84100295, "output_tokens": 3814259, "reasoning_tokens": 1704398, "cache_read_tokens": 63070995, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "tencent/hy4-preview": { "accuracy": 32.617, "latency": 2464.028, "stderr": 3.381, "cost_per_test": 0.585277, "token_totals": { "input_tokens": 181603577, "output_tokens": 28903691, "reasoning_tokens": 24008954, "cache_read_tokens": 139882560, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 32.388, "latency": 1547.695, "stderr": 3.361, "cost_per_test": 0.134306, "token_totals": { "input_tokens": 136588210, "output_tokens": 12521886, "reasoning_tokens": 8284595, "cache_read_tokens": 102894848, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 32.124, "latency": 178.329, "stderr": 3.37, "cost_per_test": 0.857214, "token_totals": { "input_tokens": 262586790, "output_tokens": 7012701, "reasoning_tokens": 4789793, "cache_read_tokens": 208172664, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 32.095, "latency": 2436.106, "stderr": 3.367, "cost_per_test": 1.552558, "token_totals": { "input_tokens": 201221502, "output_tokens": 20553651, "reasoning_tokens": 16744468, "cache_read_tokens": 130986496, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 31.922, "latency": 2973.647, "stderr": 3.352, "cost_per_test": 1.164207, "token_totals": { "input_tokens": 782193467, "output_tokens": 143162719, "reasoning_tokens": 133330984, "cache_read_tokens": 613966554, "cache_write_tokens": 168173092 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 31.55, "latency": 1297.006, "stderr": 3.35, "cost_per_test": 0.138062, "token_totals": { "input_tokens": 194143568, "output_tokens": 28577937, "reasoning_tokens": 24325230, "cache_read_tokens": 139704320, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 31.088, "latency": 2681.175, "stderr": 3.34, "cost_per_test": 7.513965, "token_totals": { "input_tokens": 608937344, "output_tokens": 36375165, "reasoning_tokens": 31251074, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 31.077, "latency": 1845.939, "stderr": 3.339, "cost_per_test": 2.972904, "token_totals": { "input_tokens": 638768422, "output_tokens": 13714742, "reasoning_tokens": 9058386, "cache_read_tokens": 574529181, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 30.894, "latency": 1333.825, "stderr": 3.323, "cost_per_test": 0.81533, "token_totals": { "input_tokens": 380333007, "output_tokens": 36121135, "reasoning_tokens": 30086863, "cache_read_tokens": 313713664, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 30.391, "latency": 3438.427, "stderr": 3.309, "cost_per_test": 2.72124, "token_totals": { "input_tokens": 561529891, "output_tokens": 26978904, "reasoning_tokens": null, "cache_read_tokens": 287912448, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 30.035, "latency": 3630.72, "stderr": 3.307, "cost_per_test": 4.829225, "token_totals": { "input_tokens": 608154133, "output_tokens": 46971256, "reasoning_tokens": 41166217, "cache_read_tokens": 508089082, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 29.999, "latency": 479.2, "stderr": 3.303, "cost_per_test": 1.68964, "token_totals": { "input_tokens": 58571818, "output_tokens": 7476677, "reasoning_tokens": 5120560, "cache_read_tokens": 39455480, "cache_write_tokens": 19099407 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 29.374, "latency": 1158.894, "stderr": 3.248, "cost_per_test": 1.794774, "token_totals": { "input_tokens": 187222743, "output_tokens": 21888011, "reasoning_tokens": 10753790, "cache_read_tokens": 148056139, "cache_write_tokens": 39133536 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 28.507, "latency": 1038.928, "stderr": 3.14, "cost_per_test": 0.048982, "token_totals": { "input_tokens": 168296091, "output_tokens": 12132305, "reasoning_tokens": 7352234, "cache_read_tokens": 127944128, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 28.069, "latency": 337.343, "stderr": 3.158, "cost_per_test": 0.156388, "token_totals": { "input_tokens": 118557354, "output_tokens": 14769391, "reasoning_tokens": 11893686, "cache_read_tokens": 97366102, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 27.273, "latency": 1288.515, "stderr": 3.113, "cost_per_test": 0.55338, "token_totals": { "input_tokens": 311523716, "output_tokens": 16401077, "reasoning_tokens": 0, "cache_read_tokens": 262109696, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 26.756, "latency": 1596.755, "stderr": 3.093, "cost_per_test": 0.729136, "token_totals": { "input_tokens": 172298347, "output_tokens": 21059828, "reasoning_tokens": 17608474, "cache_read_tokens": 133313152, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 26.63, "latency": 349.076, "stderr": 3.141, "cost_per_test": 0.134662, "token_totals": { "input_tokens": 114496920, "output_tokens": 15566679, "reasoning_tokens": 12729723, "cache_read_tokens": 91970688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 26.321, "latency": 701.825, "stderr": 2.653, "cost_per_test": 0.883559, "token_totals": { "input_tokens": 413690950, "output_tokens": 3571385, "reasoning_tokens": 1667757, "cache_read_tokens": 329680688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 25.84, "latency": 707.307, "stderr": 3.154, "cost_per_test": 1.193501, "token_totals": { "input_tokens": 116241036, "output_tokens": 7423691, "reasoning_tokens": 4447812, "cache_read_tokens": 91848017, "cache_write_tokens": 24342491 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 25.17, "latency": 1244.635, "stderr": 3.058, "cost_per_test": 0.719152, "token_totals": { "input_tokens": 171099768, "output_tokens": 21311454, "reasoning_tokens": 17772506, "cache_read_tokens": 134364160, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 24.855, "latency": 283.566, "stderr": 3.118, "cost_per_test": 0.473604, "token_totals": { "input_tokens": 84693340, "output_tokens": 3280682, "reasoning_tokens": 949587, "cache_read_tokens": 57450112, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 24.815, "latency": 2565.695, "stderr": 3.113, "cost_per_test": 0.465359, "token_totals": { "input_tokens": 585632572, "output_tokens": 50154883, "reasoning_tokens": 43456905, "cache_read_tokens": 483082516, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 23.738, "latency": 279.035, "stderr": 3.063, "cost_per_test": 1.274806, "token_totals": { "input_tokens": 82936206, "output_tokens": 4070683, "reasoning_tokens": 1866988, "cache_read_tokens": 65036497, "cache_write_tokens": 17803026 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 22.798, "latency": 87.24, "stderr": 3.028, "cost_per_test": 0.483978, "token_totals": { "input_tokens": 104490097, "output_tokens": 3925073, "reasoning_tokens": 2390046, "cache_read_tokens": 68715157, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 21.762, "latency": 951.811, "stderr": 2.978, "cost_per_test": 4.073277, "token_totals": { "input_tokens": 350470189, "output_tokens": 12925648, "reasoning_tokens": 10017673, "cache_read_tokens": 301365504, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 21.754, "latency": 1213.878, "stderr": 2.977, "cost_per_test": 1.323601, "token_totals": { "input_tokens": 149308416, "output_tokens": 12717430, "reasoning_tokens": 9488118, "cache_read_tokens": 103439338, "cache_write_tokens": 45805894 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 21.652, "latency": 233.001, "stderr": 2.963, "cost_per_test": 0.859679, "token_totals": { "input_tokens": 335113206, "output_tokens": 5873488, "reasoning_tokens": 4405887, "cache_read_tokens": 288602367, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.717, "latency": 900.586, "stderr": 2.924, "cost_per_test": 5.7963, "token_totals": { "input_tokens": 115812555, "output_tokens": 12032734, "reasoning_tokens": 9165100, "cache_read_tokens": 71528751, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 20.599, "latency": 333.833, "stderr": 2.908, "cost_per_test": 0.025079, "token_totals": { "input_tokens": 142160419, "output_tokens": 8349782, "reasoning_tokens": null, "cache_read_tokens": 108172032, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.588, "latency": 314.575, "stderr": 2.874, "cost_per_test": 0.164063, "token_totals": { "input_tokens": 85169988, "output_tokens": 4820320, "reasoning_tokens": 176867, "cache_read_tokens": 22014292, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 19.947, "latency": 1995.736, "stderr": 2.866, "cost_per_test": 0.190956, "token_totals": { "input_tokens": 449103207, "output_tokens": 40664272, "reasoning_tokens": 37199155, "cache_read_tokens": 368052701, "cache_write_tokens": 80918209 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 18.038, "latency": 105.32, "stderr": 2.766, "cost_per_test": 0.356293, "token_totals": { "input_tokens": 67586371, "output_tokens": 3157887, "reasoning_tokens": 2191235, "cache_read_tokens": 41703106, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 15.832, "latency": 209.726, "stderr": 2.617, "cost_per_test": 0.435494, "token_totals": { "input_tokens": 66735457, "output_tokens": 3616925, "reasoning_tokens": 2247001, "cache_read_tokens": 48851200, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 15.389, "latency": 385.252, "stderr": 2.365, "cost_per_test": 0.091767, "token_totals": { "input_tokens": 107926904, "output_tokens": 7430357, "reasoning_tokens": null, "cache_read_tokens": 98264704, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 15.291, "latency": 810.557, "stderr": 2.423, "cost_per_test": 0.353757, "token_totals": { "input_tokens": 147459631, "output_tokens": 7269937, "reasoning_tokens": null, "cache_read_tokens": 130876928, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-sol": { "accuracy": 15.002, "latency": 1052.036, "stderr": 2.575, "cost_per_test": 2.277613, "token_totals": { "input_tokens": 386182808, "output_tokens": 18761287, "reasoning_tokens": 15767772, "cache_read_tokens": 316266542, "cache_write_tokens": 69503073 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 12.404, "latency": 924.425, "stderr": 2.376, "cost_per_test": 0.565511, "token_totals": { "input_tokens": 247358580, "output_tokens": 8695317, "reasoning_tokens": 6542577, "cache_read_tokens": 203326968, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 11.843, "latency": 100.623, "stderr": 2.324, "cost_per_test": 0.100699, "token_totals": { "input_tokens": 39222064, "output_tokens": 3147091, "reasoning_tokens": 2378293, "cache_read_tokens": 21372192, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 11.503, "latency": 376.514, "stderr": 1.957, "cost_per_test": 0.040042, "token_totals": { "input_tokens": 66417509, "output_tokens": 2436638, "reasoning_tokens": 1177368, "cache_read_tokens": 53971712, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 10.466, "latency": 310.263, "stderr": 1.993, "cost_per_test": 0.093885, "token_totals": { "input_tokens": 83119598, "output_tokens": 4291556, "reasoning_tokens": 2832859, "cache_read_tokens": 61095936, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 9.295, "latency": 100.956, "stderr": 2.092, "cost_per_test": 0.301555, "token_totals": { "input_tokens": 33380072, "output_tokens": 1903235, "reasoning_tokens": 1460348, "cache_read_tokens": 17443775, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 8.808, "latency": 1006.185, "stderr": 2.045, "cost_per_test": 0.806171, "token_totals": { "input_tokens": 201253946, "output_tokens": 26480732, "reasoning_tokens": 25245242, "cache_read_tokens": 169648384, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 7.772, "latency": 500.826, "stderr": 1.779, "cost_per_test": 0.017023, "token_totals": { "input_tokens": 114032276, "output_tokens": 3002594, "reasoning_tokens": 1387062, "cache_read_tokens": 98540288, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 7.679, "latency": 378.159, "stderr": 1.81, "cost_per_test": 0.298017, "token_totals": { "input_tokens": 94733985, "output_tokens": 3383446, "reasoning_tokens": 2053059, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 7.254, "latency": 47.461, "stderr": 1.872, "cost_per_test": 0.081536, "token_totals": { "input_tokens": 74874616, "output_tokens": 1868593, "reasoning_tokens": 1185933, "cache_read_tokens": 42212661, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 6.952, "latency": 89.262, "stderr": 1.75, "cost_per_test": 0.119262, "token_totals": { "input_tokens": 19969068, "output_tokens": 1606938, "reasoning_tokens": 960969, "cache_read_tokens": 14756820, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 6.872, "latency": 302.854, "stderr": 1.677, "cost_per_test": 0.120058, "token_totals": { "input_tokens": 74679791, "output_tokens": 2724867, "reasoning_tokens": 1450884, "cache_read_tokens": 10592268, "cache_write_tokens": 4764496 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.3": { "accuracy": 6.179, "latency": 97.519, "stderr": 1.732, "cost_per_test": 0.116616, "token_totals": { "input_tokens": 37643889, "output_tokens": 1781754, "reasoning_tokens": 1092969, "cache_read_tokens": 27621238, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 5.855, "latency": 155.016, "stderr": 1.537, "cost_per_test": 0.226416, "token_totals": { "input_tokens": 73419285, "output_tokens": 2773686, "reasoning_tokens": 1184967, "cache_read_tokens": 53705606, "cache_write_tokens": 18982650 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 4.974, "latency": 220.306, "stderr": 1.417, "cost_per_test": 0.069662, "token_totals": { "input_tokens": 154740627, "output_tokens": 4455634, "reasoning_tokens": 3038667, "cache_read_tokens": 128182784, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 4.482, "latency": 430.673, "stderr": 1.47, "cost_per_test": 1.443655, "token_totals": { "input_tokens": 150690955, "output_tokens": 7011852, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 3.109, "latency": 18.744, "stderr": 1.253, "cost_per_test": 0.014308, "token_totals": { "input_tokens": 6757967, "output_tokens": 906826, "reasoning_tokens": 487639, "cache_read_tokens": 1281568, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 2.591, "latency": 1180.079, "stderr": 1.146, "cost_per_test": 0.030533, "token_totals": { "input_tokens": 69555925, "output_tokens": 22469385, "reasoning_tokens": 14413630, "cache_read_tokens": 51967923, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 2.487, "latency": 29.67, "stderr": 1.018, "cost_per_test": 0.023633, "token_totals": { "input_tokens": 43610163, "output_tokens": 1545275, "reasoning_tokens": 1248847, "cache_read_tokens": 29554557, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "fact_pattern_analysis": { "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 55.767, "latency": 1324.466, "stderr": 7.657, "cost_per_test": 0.460654, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 51.163, "latency": 1615.462, "stderr": 7.713, "cost_per_test": 11.625261, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 48.837, "latency": 962.376, "stderr": 7.713, "cost_per_test": 4.479733, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 46.512, "latency": 1141.924, "stderr": 7.696, "cost_per_test": 2.157843, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 46.512, "latency": 3840.498, "stderr": 7.696, "cost_per_test": 14.475041, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 45.814, "latency": 626.149, "stderr": 7.612, "cost_per_test": 0.340841, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 43.721, "latency": 260.736, "stderr": 7.344, "cost_per_test": 0.392251, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 39.535, "latency": 504.76, "stderr": 7.544, "cost_per_test": 1.775974, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 39.535, "latency": 812.519, "stderr": 7.544, "cost_per_test": 2.8502, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 39.535, "latency": 2074.148, "stderr": 7.544, "cost_per_test": 3.85373, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 39.535, "latency": 3212.209, "stderr": 7.544, "cost_per_test": 7.966557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 39.433, "latency": 879.948, "stderr": 7.525, "cost_per_test": 6.080646, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 38.981, "latency": 1258.298, "stderr": 7.195, "cost_per_test": 0.050237, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 37.209, "latency": 3662.48, "stderr": 7.458, "cost_per_test": 4.593033, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 34.884, "latency": 169.217, "stderr": 7.354, "cost_per_test": 0.792812, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 34.884, "latency": 946.959, "stderr": 7.354, "cost_per_test": 1.653949, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 34.884, "latency": 1377.571, "stderr": 7.354, "cost_per_test": 0.834749, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 34.884, "latency": 3230.755, "stderr": 7.354, "cost_per_test": 1.851233, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 32.558, "latency": 125.373, "stderr": 5.387, "cost_per_test": 0.261785, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 32.558, "latency": 2637.671, "stderr": 7.231, "cost_per_test": 3.173861, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 31.86, "latency": 1302.263, "stderr": 7.108, "cost_per_test": 0.530547, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 31.83, "latency": 4183.973, "stderr": 7.101, "cost_per_test": 4.077226, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 30.233, "latency": 1028.913, "stderr": 7.087, "cost_per_test": 1.20312, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 30.233, "latency": 1869.987, "stderr": 7.087, "cost_per_test": 0.133577, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 30.179, "latency": 2393.442, "stderr": 7.074, "cost_per_test": 0.531712, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 30.158, "latency": 2681.727, "stderr": 7.069, "cost_per_test": 1.15627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 29.535, "latency": 2051.965, "stderr": 6.956, "cost_per_test": 0.212665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 28.14, "latency": 1686.83, "stderr": 6.683, "cost_per_test": 0.629486, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 27.907, "latency": 838.059, "stderr": 6.921, "cost_per_test": 5.184223, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 27.907, "latency": 956.476, "stderr": 6.921, "cost_per_test": 3.881333, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 27.907, "latency": 2653.408, "stderr": 6.921, "cost_per_test": 1.716952, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 27.82, "latency": 721.879, "stderr": 6.9, "cost_per_test": 0.967508, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 27.8, "latency": 1664.779, "stderr": 6.895, "cost_per_test": 0.163393, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 27.209, "latency": 408.316, "stderr": 6.782, "cost_per_test": 0.136181, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 27.14, "latency": 1273.988, "stderr": 6.764, "cost_per_test": 1.783178, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 26.744, "latency": 995.717, "stderr": 5.404, "cost_per_test": 2.528872, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 25.581, "latency": 518.318, "stderr": 6.733, "cost_per_test": 1.616173, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 25.581, "latency": 3047.67, "stderr": 6.733, "cost_per_test": 0.549062, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 24.562, "latency": 954.507, "stderr": 6.182, "cost_per_test": 0.42429, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 24.186, "latency": 348.287, "stderr": 6.429, "cost_per_test": 0.147096, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 23.256, "latency": 845.013, "stderr": 6.519, "cost_per_test": 1.284935, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 23.256, "latency": 2784.942, "stderr": 6.519, "cost_per_test": 7.969384, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 21.86, "latency": 1278.557, "stderr": 6.192, "cost_per_test": 0.651612, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 20.93, "latency": 309.359, "stderr": 6.277, "cost_per_test": 0.394055, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 20.853, "latency": 337.343, "stderr": 6.254, "cost_per_test": 1.387812, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 18.605, "latency": 87.896, "stderr": 6.005, "cost_per_test": 0.427212, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 16.279, "latency": 110.876, "stderr": 5.696, "cost_per_test": 0.342251, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 16.279, "latency": 237.175, "stderr": 5.696, "cost_per_test": 0.753889, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 16.279, "latency": 939.613, "stderr": 5.696, "cost_per_test": 2.094899, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 14.186, "latency": 368.229, "stderr": 5.049, "cost_per_test": 0.166429, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 13.776, "latency": 224.965, "stderr": 5.28, "cost_per_test": 0.419675, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 13.768, "latency": 386.935, "stderr": 5.277, "cost_per_test": 0.025975, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 12.558, "latency": 495.424, "stderr": 4.881, "cost_per_test": 0.104177, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 10.465, "latency": 426.904, "stderr": 4.061, "cost_per_test": 0.037953, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 9.302, "latency": 110.173, "stderr": 4.482, "cost_per_test": 0.093878, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 9.302, "latency": 980.497, "stderr": 4.482, "cost_per_test": 0.607497, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 7.907, "latency": 221.457, "stderr": 3.875, "cost_per_test": 0.070223, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 7.209, "latency": 641.995, "stderr": 3.527, "cost_per_test": 0.01803, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 6.977, "latency": 115.164, "stderr": 3.931, "cost_per_test": 0.274484, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 6.977, "latency": 874.453, "stderr": 3.931, "cost_per_test": 0.736955, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 6.279, "latency": 178.603, "stderr": 3.585, "cost_per_test": 0.219905, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 6.279, "latency": 356.11, "stderr": 3.585, "cost_per_test": 0.09907, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 5.581, "latency": 89.445, "stderr": 3.197, "cost_per_test": 0.120294, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 5.482, "latency": 455.125, "stderr": 3.128, "cost_per_test": 0.364885, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 4.651, "latency": 23.467, "stderr": 3.249, "cost_per_test": 0.015379, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 4.651, "latency": 60.341, "stderr": 3.249, "cost_per_test": 0.086864, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 4.477, "latency": 96.392, "stderr": 3.13, "cost_per_test": 0.092552, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 3.953, "latency": 314.548, "stderr": 2.807, "cost_per_test": 0.114892, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "inception/mercury-2.5": { "accuracy": 2.326, "latency": 34.135, "stderr": 2.326, "cost_per_test": 0.024313, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 2.326, "latency": 532.451, "stderr": 2.326, "cost_per_test": 1.882853, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1584.808, "stderr": 0.0, "cost_per_test": 0.034737, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "rule_source_lookup": { "anthropic/claude-fable-5-1": { "accuracy": 66.667, "latency": 1590.623, "stderr": 8.754, "cost_per_test": 11.284536, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 63.333, "latency": 195.725, "stderr": 8.949, "cost_per_test": 0.307916, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 60.0, "latency": 2863.402, "stderr": 9.097, "cost_per_test": 1.698419, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 56.667, "latency": 818.772, "stderr": 9.202, "cost_per_test": 3.044362, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 56.667, "latency": 950.785, "stderr": 9.202, "cost_per_test": 5.207646, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 50.0, "latency": 972.116, "stderr": 9.285, "cost_per_test": 1.973979, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 50.0, "latency": 1257.166, "stderr": 9.285, "cost_per_test": 0.750516, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 49.962, "latency": 3532.864, "stderr": 9.278, "cost_per_test": 13.741037, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 46.667, "latency": 1361.742, "stderr": 9.264, "cost_per_test": 0.492773, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 46.667, "latency": 3858.029, "stderr": 9.264, "cost_per_test": 9.831783, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 46.583, "latency": 747.145, "stderr": 9.248, "cost_per_test": 5.813849, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 45.554, "latency": 2822.295, "stderr": 9.096, "cost_per_test": 1.085204, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 43.667, "latency": 103.014, "stderr": 7.311, "cost_per_test": 0.228706, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 43.333, "latency": 419.046, "stderr": 9.202, "cost_per_test": 0.728366, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 43.333, "latency": 446.798, "stderr": 9.202, "cost_per_test": 1.712984, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 43.333, "latency": 489.476, "stderr": 9.202, "cost_per_test": 0.812812, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 43.333, "latency": 1574.775, "stderr": 9.202, "cost_per_test": 1.540859, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 43.256, "latency": 2077.708, "stderr": 9.186, "cost_per_test": 1.497169, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 43.129, "latency": 1043.928, "stderr": 9.159, "cost_per_test": 1.678605, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 42.59, "latency": 339.091, "stderr": 8.625, "cost_per_test": 0.163641, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 42.333, "latency": 430.403, "stderr": 7.591, "cost_per_test": 0.193793, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 41.667, "latency": 1297.966, "stderr": 8.47, "cost_per_test": 0.552303, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 40.0, "latency": 116.279, "stderr": 9.097, "cost_per_test": 0.471874, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.0, "latency": 413.788, "stderr": 9.097, "cost_per_test": 0.261741, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 40.0, "latency": 919.518, "stderr": 9.097, "cost_per_test": 1.617708, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 39.929, "latency": 1094.178, "stderr": 9.081, "cost_per_test": 0.110782, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 39.714, "latency": 151.907, "stderr": 9.036, "cost_per_test": 0.788644, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 39.652, "latency": 302.327, "stderr": 9.022, "cost_per_test": 0.112383, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 37.933, "latency": 851.616, "stderr": 8.725, "cost_per_test": 0.043707, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 37.909, "latency": 1369.988, "stderr": 8.72, "cost_per_test": 0.668801, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 36.667, "latency": 153.734, "stderr": 8.949, "cost_per_test": 0.109333, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 36.667, "latency": 162.06, "stderr": 8.949, "cost_per_test": 0.789933, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 36.667, "latency": 342.933, "stderr": 8.949, "cost_per_test": 1.290724, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 36.667, "latency": 1875.339, "stderr": 8.949, "cost_per_test": 0.341665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 36.667, "latency": 2586.868, "stderr": 8.949, "cost_per_test": 0.636639, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 34.667, "latency": 1210.271, "stderr": 8.56, "cost_per_test": 0.724403, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 33.333, "latency": 2255.415, "stderr": 8.754, "cost_per_test": 1.26074, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 33.333, "latency": 2904.931, "stderr": 8.754, "cost_per_test": 4.09629, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 33.233, "latency": 246.939, "stderr": 8.728, "cost_per_test": 0.021168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.5": { "accuracy": 33.233, "latency": 306.772, "stderr": 8.728, "cost_per_test": 0.387411, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 32.958, "latency": 1383.783, "stderr": 8.659, "cost_per_test": 0.119899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 30.0, "latency": 2236.425, "stderr": 8.51, "cost_per_test": 5.48599, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 26.667, "latency": 833.412, "stderr": 8.212, "cost_per_test": 1.158281, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 23.333, "latency": 126.702, "stderr": 7.854, "cost_per_test": 0.286683, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-astra": { "accuracy": 23.333, "latency": 705.261, "stderr": 7.854, "cost_per_test": 4.662827, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 23.333, "latency": 799.357, "stderr": 7.854, "cost_per_test": 3.524307, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 23.167, "latency": 94.714, "stderr": 7.8, "cost_per_test": 0.315209, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 23.097, "latency": 227.911, "stderr": 7.776, "cost_per_test": 0.71482, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 20.0, "latency": 945.988, "stderr": 7.428, "cost_per_test": 1.02288, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 20.0, "latency": 1517.938, "stderr": 7.428, "cost_per_test": 0.123429, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 19.667, "latency": 165.068, "stderr": 6.13, "cost_per_test": 0.02233, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 17.0, "latency": 199.19, "stderr": 6.435, "cost_per_test": 0.073258, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 16.667, "latency": 923.305, "stderr": 6.92, "cost_per_test": 1.829993, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 16.5, "latency": 77.219, "stderr": 6.853, "cost_per_test": 0.085031, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 14.485, "latency": 391.772, "stderr": 6.107, "cost_per_test": 0.215865, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 14.0, "latency": 222.832, "stderr": 5.199, "cost_per_test": 0.067974, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 13.333, "latency": 41.274, "stderr": 6.312, "cost_per_test": 0.060332, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 13.333, "latency": 86.667, "stderr": 6.312, "cost_per_test": 0.246919, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 12.667, "latency": 231.03, "stderr": 5.338, "cost_per_test": 0.092243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 10.333, "latency": 273.171, "stderr": 4.971, "cost_per_test": 0.206147, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 10.0, "latency": 678.113, "stderr": 5.571, "cost_per_test": 0.417333, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 10.0, "latency": 687.26, "stderr": 5.571, "cost_per_test": 0.560241, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 10.0, "latency": 732.305, "stderr": 5.571, "cost_per_test": 0.020854, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 8.0, "latency": 78.155, "stderr": 4.533, "cost_per_test": 0.102575, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 8.0, "latency": 126.008, "stderr": 4.533, "cost_per_test": 0.209741, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 8.0, "latency": 149.816, "stderr": 4.533, "cost_per_test": 0.05178, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 7.0, "latency": 366.583, "stderr": 3.9, "cost_per_test": 0.014016, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.3": { "accuracy": 6.667, "latency": 85.502, "stderr": 4.632, "cost_per_test": 0.097472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 6.667, "latency": 422.308, "stderr": 4.632, "cost_per_test": 1.333021, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 3.333, "latency": 18.573, "stderr": 3.333, "cost_per_test": 0.010765, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 2.333, "latency": 29.325, "stderr": 2.333, "cost_per_test": 0.021089, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "current_temporal_analysis": { "google/gemini-4-argon": { "accuracy": 66.667, "latency": 1606.495, "stderr": 8.754, "cost_per_test": 5.143482, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 59.816, "latency": 2244.633, "stderr": 9.071, "cost_per_test": 16.518066, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 56.614, "latency": 1369.127, "stderr": 9.193, "cost_per_test": 2.742275, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 56.386, "latency": 1540.94, "stderr": 9.158, "cost_per_test": 0.541685, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 53.222, "latency": 4008.358, "stderr": 9.245, "cost_per_test": 2.275751, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 53.169, "latency": 1206.807, "stderr": 9.236, "cost_per_test": 9.205921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 53.122, "latency": 1758.869, "stderr": 9.229, "cost_per_test": 0.154282, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder": { "accuracy": 50.0, "latency": 653.458, "stderr": 9.285, "cost_per_test": 2.232327, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 50.0, "latency": 3691.615, "stderr": 9.285, "cost_per_test": 9.388335, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 49.933, "latency": 4722.823, "stderr": 9.273, "cost_per_test": 11.848978, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 49.929, "latency": 2562.167, "stderr": 9.272, "cost_per_test": 3.703546, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 49.889, "latency": 3069.918, "stderr": 9.265, "cost_per_test": 1.903846, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 49.879, "latency": 1207.608, "stderr": 9.263, "cost_per_test": 6.007301, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 49.769, "latency": 660.075, "stderr": 9.245, "cost_per_test": 0.354536, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 46.667, "latency": 1363.832, "stderr": 9.264, "cost_per_test": 2.443799, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 46.56, "latency": 4157.048, "stderr": 9.243, "cost_per_test": 2.23471, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 46.558, "latency": 5343.646, "stderr": 9.243, "cost_per_test": 19.635953, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 46.524, "latency": 584.037, "stderr": 9.237, "cost_per_test": 1.945154, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 45.667, "latency": 3066.144, "stderr": 9.118, "cost_per_test": 3.074284, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 43.556, "latency": 2180.108, "stderr": 8.784, "cost_per_test": 0.952634, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 43.333, "latency": 228.113, "stderr": 9.202, "cost_per_test": 1.013323, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 43.333, "latency": 812.418, "stderr": 9.202, "cost_per_test": 1.209783, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 43.197, "latency": 1494.556, "stderr": 9.174, "cost_per_test": 0.158431, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 43.088, "latency": 1947.331, "stderr": 9.151, "cost_per_test": 1.177468, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 42.99, "latency": 446.097, "stderr": 9.132, "cost_per_test": 0.208159, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 42.333, "latency": 326.12, "stderr": 9.042, "cost_per_test": 0.455908, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 39.667, "latency": 152.954, "stderr": 6.441, "cost_per_test": 0.290519, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 39.0, "latency": 397.458, "stderr": 8.923, "cost_per_test": 0.145689, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 36.667, "latency": 331.714, "stderr": 8.949, "cost_per_test": 0.601971, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 36.667, "latency": 3939.143, "stderr": 8.949, "cost_per_test": 1.419874, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 36.595, "latency": 1727.127, "stderr": 8.931, "cost_per_test": 1.136219, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 36.593, "latency": 1328.168, "stderr": 8.931, "cost_per_test": 0.058099, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 36.58, "latency": 3258.22, "stderr": 8.928, "cost_per_test": 0.762342, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 36.442, "latency": 326.113, "stderr": 8.895, "cost_per_test": 1.386875, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 36.417, "latency": 372.31, "stderr": 8.889, "cost_per_test": 0.027631, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 33.283, "latency": 1908.876, "stderr": 8.741, "cost_per_test": 2.13313, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 33.222, "latency": 4946.592, "stderr": 8.725, "cost_per_test": 6.593928, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 33.151, "latency": 910.516, "stderr": 8.707, "cost_per_test": 1.404458, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 32.978, "latency": 3317.224, "stderr": 8.664, "cost_per_test": 0.575904, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 31.123, "latency": 1734.264, "stderr": 8.271, "cost_per_test": 0.698028, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 30.0, "latency": 112.732, "stderr": 8.51, "cost_per_test": 0.562793, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 30.0, "latency": 1517.93, "stderr": 8.51, "cost_per_test": 2.342482, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 29.944, "latency": 1384.946, "stderr": 8.494, "cost_per_test": 9.278145, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 29.653, "latency": 477.413, "stderr": 8.414, "cost_per_test": 0.214205, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "meta/muse-spark-open": { "accuracy": 28.0, "latency": 760.495, "stderr": 6.368, "cost_per_test": 0.288385, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 26.667, "latency": 1352.021, "stderr": 8.212, "cost_per_test": 5.233081, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 26.467, "latency": 1213.636, "stderr": 8.152, "cost_per_test": 0.756993, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 26.444, "latency": 279.898, "stderr": 8.146, "cost_per_test": 0.929565, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 26.333, "latency": 502.508, "stderr": 7.071, "cost_per_test": 0.088018, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-sol": { "accuracy": 23.179, "latency": 1553.84, "stderr": 7.804, "cost_per_test": 3.706383, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 23.107, "latency": 303.119, "stderr": 7.779, "cost_per_test": 0.551759, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 19.875, "latency": 132.729, "stderr": 7.382, "cost_per_test": 0.396307, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 18.333, "latency": 398.044, "stderr": 6.29, "cost_per_test": 0.105531, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 16.542, "latency": 472.355, "stderr": 6.869, "cost_per_test": 0.301399, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 16.533, "latency": 2764.929, "stderr": 6.866, "cost_per_test": 0.233793, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 15.267, "latency": 916.798, "stderr": 5.743, "cost_per_test": 0.3676, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 15.0, "latency": 451.743, "stderr": 5.648, "cost_per_test": 0.04028, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 13.056, "latency": 130.181, "stderr": 6.183, "cost_per_test": 0.160784, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 13.024, "latency": 123.063, "stderr": 6.168, "cost_per_test": 0.099933, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 12.333, "latency": 504.95, "stderr": 5.905, "cost_per_test": 0.013685, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.3": { "accuracy": 10.0, "latency": 129.458, "stderr": 5.571, "cost_per_test": 0.13418, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 10.0, "latency": 1413.578, "stderr": 5.571, "cost_per_test": 1.066462, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 9.8, "latency": 138.829, "stderr": 5.462, "cost_per_test": 0.312752, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 6.542, "latency": 413.595, "stderr": 4.546, "cost_per_test": 0.135098, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 6.5, "latency": 420.093, "stderr": 4.518, "cost_per_test": 1.381649, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 3.333, "latency": 52.71, "stderr": 3.333, "cost_per_test": 0.075425, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 3.333, "latency": 168.511, "stderr": 3.333, "cost_per_test": 0.238473, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 3.333, "latency": 283.728, "stderr": 3.333, "cost_per_test": 0.083781, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 3.333, "latency": 903.722, "stderr": 3.333, "cost_per_test": 0.027891, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 2.333, "latency": 36.086, "stderr": 2.333, "cost_per_test": 0.022182, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 21.131, "stderr": 0.0, "cost_per_test": 0.013534, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "numbers_calculations": { "anthropic/claude-opus-5-5": { "accuracy": 70.0, "latency": 2259.551, "stderr": 8.51, "cost_per_test": 7.944016, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 60.0, "latency": 502.859, "stderr": 9.097, "cost_per_test": 2.664386, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 60.0, "latency": 526.541, "stderr": 9.097, "cost_per_test": 1.946484, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 60.0, "latency": 752.921, "stderr": 9.097, "cost_per_test": 5.30143, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 60.0, "latency": 1213.722, "stderr": 9.097, "cost_per_test": 0.2941, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 60.0, "latency": 2198.898, "stderr": 9.097, "cost_per_test": 5.159882, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 53.25, "latency": 1793.902, "stderr": 9.25, "cost_per_test": 0.539077, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 53.167, "latency": 2268.254, "stderr": 9.237, "cost_per_test": 1.236253, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 50.0, "latency": 120.595, "stderr": 9.285, "cost_per_test": 0.619008, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 50.0, "latency": 325.851, "stderr": 9.285, "cost_per_test": 2.303921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 48.667, "latency": 73.558, "stderr": 8.175, "cost_per_test": 0.149096, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 47.0, "latency": 504.122, "stderr": 8.869, "cost_per_test": 0.729707, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 46.667, "latency": 234.86, "stderr": 9.264, "cost_per_test": 0.855352, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 46.667, "latency": 250.763, "stderr": 9.264, "cost_per_test": 0.800257, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 46.667, "latency": 424.439, "stderr": 9.264, "cost_per_test": 0.595682, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 46.667, "latency": 455.508, "stderr": 9.264, "cost_per_test": 0.876776, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 46.667, "latency": 710.489, "stderr": 9.264, "cost_per_test": 0.077152, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 46.667, "latency": 783.344, "stderr": 9.264, "cost_per_test": 2.145907, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 45.667, "latency": 677.794, "stderr": 9.118, "cost_per_test": 0.241521, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 45.667, "latency": 784.284, "stderr": 9.118, "cost_per_test": 0.063015, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 45.667, "latency": 1148.921, "stderr": 9.118, "cost_per_test": 1.425651, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 43.333, "latency": 128.464, "stderr": 9.202, "cost_per_test": 0.187818, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 43.333, "latency": 343.387, "stderr": 9.202, "cost_per_test": 0.183382, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 41.333, "latency": 178.13, "stderr": 8.876, "cost_per_test": 0.071997, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder": { "accuracy": 40.0, "latency": 206.578, "stderr": 9.097, "cost_per_test": 0.792122, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 40.0, "latency": 384.892, "stderr": 9.097, "cost_per_test": 0.609108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 40.0, "latency": 540.429, "stderr": 9.097, "cost_per_test": 0.829064, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 40.0, "latency": 1742.081, "stderr": 9.097, "cost_per_test": 2.186392, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 40.0, "latency": 2328.746, "stderr": 9.097, "cost_per_test": 1.946576, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 39.667, "latency": 94.184, "stderr": 9.027, "cost_per_test": 0.306179, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 38.667, "latency": 339.074, "stderr": 7.904, "cost_per_test": 0.15679, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 38.0, "latency": 150.488, "stderr": 8.741, "cost_per_test": 0.051675, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 36.667, "latency": 142.817, "stderr": 8.949, "cost_per_test": 0.550501, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 36.667, "latency": 752.627, "stderr": 8.949, "cost_per_test": 1.027503, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 33.667, "latency": 455.905, "stderr": 8.353, "cost_per_test": 0.260482, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 33.333, "latency": 70.814, "stderr": 8.754, "cost_per_test": 0.355132, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 33.333, "latency": 148.479, "stderr": 8.754, "cost_per_test": 0.257231, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 33.333, "latency": 387.008, "stderr": 8.754, "cost_per_test": 0.381672, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 33.333, "latency": 946.298, "stderr": 8.754, "cost_per_test": 0.546571, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 32.667, "latency": 564.446, "stderr": 8.136, "cost_per_test": 0.024687, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 32.333, "latency": 687.242, "stderr": 8.546, "cost_per_test": 0.406577, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 32.333, "latency": 726.33, "stderr": 8.546, "cost_per_test": 0.320631, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 30.0, "latency": 333.476, "stderr": 8.51, "cost_per_test": 1.539682, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 30.0, "latency": 819.844, "stderr": 8.51, "cost_per_test": 0.070109, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 30.0, "latency": 1086.03, "stderr": 8.51, "cost_per_test": 0.196728, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 29.8, "latency": 185.832, "stderr": 8.455, "cost_per_test": 0.012388, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "stepfun/step-5-preview": { "accuracy": 29.0, "latency": 647.942, "stderr": 8.282, "cost_per_test": 0.235703, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-sol": { "accuracy": 26.667, "latency": 384.955, "stderr": 8.212, "cost_per_test": 0.735903, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 23.667, "latency": 269.336, "stderr": 7.42, "cost_per_test": 0.062924, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 23.333, "latency": 88.311, "stderr": 7.854, "cost_per_test": 0.082771, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 23.333, "latency": 101.165, "stderr": 7.854, "cost_per_test": 0.206993, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 23.333, "latency": 127.478, "stderr": 7.854, "cost_per_test": 0.245076, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 23.333, "latency": 542.04, "stderr": 7.854, "cost_per_test": 0.24358, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 20.0, "latency": 44.318, "stderr": 7.428, "cost_per_test": 0.059752, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.0, "latency": 258.28, "stderr": 7.428, "cost_per_test": 1.623587, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 20.0, "latency": 551.988, "stderr": 7.428, "cost_per_test": 0.395054, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 19.4, "latency": 196.912, "stderr": 7.226, "cost_per_test": 0.085401, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 17.333, "latency": 94.288, "stderr": 5.91, "cost_per_test": 0.105905, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 17.0, "latency": 178.17, "stderr": 6.435, "cost_per_test": 0.072386, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 17.0, "latency": 311.354, "stderr": 6.435, "cost_per_test": 0.009895, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling": { "accuracy": 17.0, "latency": 466.325, "stderr": 6.435, "cost_per_test": 0.214027, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 16.667, "latency": 56.426, "stderr": 6.92, "cost_per_test": 0.056398, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 16.333, "latency": 198.363, "stderr": 5.498, "cost_per_test": 0.046584, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 16.333, "latency": 287.096, "stderr": 5.498, "cost_per_test": 0.021077, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 15.667, "latency": 52.646, "stderr": 6.569, "cost_per_test": 0.060833, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 11.333, "latency": 180.956, "stderr": 5.461, "cost_per_test": 0.128621, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 10.0, "latency": 20.929, "stderr": 5.571, "cost_per_test": 0.013865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 9.0, "latency": 224.875, "stderr": 5.082, "cost_per_test": 0.68906, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 8.0, "latency": 22.004, "stderr": 4.533, "cost_per_test": 0.015262, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 7.0, "latency": 116.279, "stderr": 3.9, "cost_per_test": 0.028897, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 3.333, "latency": 978.65, "stderr": 3.333, "cost_per_test": 0.024076, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "forms_filings": { "anthropic/claude-sonnet-5-5": { "accuracy": 40.0, "latency": 3778.699, "stderr": 9.097, "cost_per_test": 10.250628, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 33.333, "latency": 1400.675, "stderr": 8.754, "cost_per_test": 0.547143, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 33.185, "latency": 1353.851, "stderr": 8.716, "cost_per_test": 7.081583, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 30.0, "latency": 1194.065, "stderr": 8.51, "cost_per_test": 2.217268, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 30.0, "latency": 1264.709, "stderr": 8.51, "cost_per_test": 2.488422, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 30.0, "latency": 2185.691, "stderr": 8.51, "cost_per_test": 17.995831, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 30.0, "latency": 4397.815, "stderr": 8.51, "cost_per_test": 17.005457, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 26.667, "latency": 1114.894, "stderr": 8.212, "cost_per_test": 9.612732, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 26.667, "latency": 1123.248, "stderr": 8.212, "cost_per_test": 4.046075, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 26.667, "latency": 2805.95, "stderr": 8.212, "cost_per_test": 0.680751, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 26.667, "latency": 3738.849, "stderr": 8.212, "cost_per_test": 2.807198, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 25.333, "latency": 164.491, "stderr": 6.776, "cost_per_test": 0.387308, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 24.667, "latency": 310.003, "stderr": 7.697, "cost_per_test": 0.517794, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 23.333, "latency": 551.297, "stderr": 7.854, "cost_per_test": 2.329763, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 23.333, "latency": 800.747, "stderr": 7.854, "cost_per_test": 1.21292, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 23.333, "latency": 1583.572, "stderr": 7.854, "cost_per_test": 0.163853, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 207.207, "stderr": 7.428, "cost_per_test": 1.032835, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 20.0, "latency": 2800.773, "stderr": 7.428, "cost_per_test": 4.479058, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 3013.858, "stderr": 7.428, "cost_per_test": 1.836345, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 3072.683, "stderr": 7.428, "cost_per_test": 3.682319, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 3219.385, "stderr": 7.428, "cost_per_test": 1.938554, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 20.0, "latency": 3272.354, "stderr": 7.428, "cost_per_test": 1.431486, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 20.0, "latency": 3672.354, "stderr": 7.428, "cost_per_test": 11.447471, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 19.947, "latency": 2048.707, "stderr": 7.408, "cost_per_test": 0.187946, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 19.944, "latency": 1388.167, "stderr": 7.407, "cost_per_test": 2.360426, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 19.875, "latency": 356.103, "stderr": 7.382, "cost_per_test": 0.235599, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 18.0, "latency": 635.944, "stderr": 6.787, "cost_per_test": 0.399484, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 17.0, "latency": 1141.66, "stderr": 6.435, "cost_per_test": 0.061032, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 16.667, "latency": 292.436, "stderr": 6.92, "cost_per_test": 0.990018, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 16.667, "latency": 339.21, "stderr": 6.92, "cost_per_test": 0.669171, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 16.667, "latency": 1488.257, "stderr": 6.92, "cost_per_test": 0.881545, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 16.667, "latency": 4910.699, "stderr": 6.92, "cost_per_test": 6.681535, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 15.667, "latency": 400.472, "stderr": 6.569, "cost_per_test": 0.181934, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 14.667, "latency": 1628.156, "stderr": 6.191, "cost_per_test": 0.711742, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 13.333, "latency": 638.305, "stderr": 6.312, "cost_per_test": 2.528215, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 13.333, "latency": 1588.793, "stderr": 6.312, "cost_per_test": 0.867778, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 13.333, "latency": 3355.896, "stderr": 6.312, "cost_per_test": 0.615879, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 12.795, "latency": 2369.492, "stderr": 6.076, "cost_per_test": 0.262741, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 12.333, "latency": 520.504, "stderr": 5.905, "cost_per_test": 0.215479, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 12.333, "latency": 1812.809, "stderr": 5.905, "cost_per_test": 1.03018, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 11.667, "latency": 848.017, "stderr": 4.844, "cost_per_test": 0.290359, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 10.333, "latency": 424.78, "stderr": 4.971, "cost_per_test": 0.118047, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 10.0, "latency": 125.77, "stderr": 5.571, "cost_per_test": 0.587181, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 10.0, "latency": 254.763, "stderr": 5.571, "cost_per_test": 0.589702, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 10.0, "latency": 1233.388, "stderr": 5.571, "cost_per_test": 5.638196, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 6.667, "latency": 120.841, "stderr": 4.632, "cost_per_test": 0.1451, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 6.667, "latency": 135.418, "stderr": 4.632, "cost_per_test": 0.425256, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 6.667, "latency": 388.616, "stderr": 4.632, "cost_per_test": 1.955097, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 6.667, "latency": 443.722, "stderr": 4.632, "cost_per_test": 0.032484, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling": { "accuracy": 6.667, "latency": 1231.408, "stderr": 4.632, "cost_per_test": 0.493899, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-astra": { "accuracy": 6.667, "latency": 1243.963, "stderr": 4.632, "cost_per_test": 8.333408, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 6.417, "latency": 906.405, "stderr": 4.462, "cost_per_test": 1.690921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 4.667, "latency": 547.636, "stderr": 3.242, "cost_per_test": 0.075732, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 3.333, "latency": 139.219, "stderr": 3.333, "cost_per_test": 0.438674, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 3.333, "latency": 392.718, "stderr": 3.333, "cost_per_test": 0.134966, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 3.333, "latency": 639.986, "stderr": 3.333, "cost_per_test": 0.02572, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 3.333, "latency": 1222.959, "stderr": 3.333, "cost_per_test": 0.78066, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 3.333, "latency": 1635.017, "stderr": 3.333, "cost_per_test": 1.295762, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 3.333, "latency": 1823.192, "stderr": 3.333, "cost_per_test": 1.949409, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 24.412, "stderr": 0.0, "cost_per_test": 0.017174, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 41.899, "stderr": 0.0, "cost_per_test": 0.031988, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 75.867, "stderr": 0.0, "cost_per_test": 0.112503, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 112.756, "stderr": 0.0, "cost_per_test": 0.147642, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 136.347, "stderr": 0.0, "cost_per_test": 0.174303, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 226.784, "stderr": 0.0, "cost_per_test": 0.325413, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 357.11, "stderr": 0.0, "cost_per_test": 0.105246, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 431.478, "stderr": 0.0, "cost_per_test": 0.127614, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 511.366, "stderr": 0.0, "cost_per_test": 0.441983, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 675.993, "stderr": 0.0, "cost_per_test": 1.974378, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 800.005, "stderr": 0.0, "cost_per_test": 0.028078, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 1481.667, "stderr": 0.0, "cost_per_test": 3.18507, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "controversy_precedence_analysis": { "meta/muse_spark_1_3_max": { "accuracy": 36.667, "latency": 330.347, "stderr": 8.949, "cost_per_test": 0.476801, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 30.0, "latency": 4856.106, "stderr": 8.51, "cost_per_test": 18.106388, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 26.667, "latency": 1309.206, "stderr": 8.212, "cost_per_test": 2.63952, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 26.667, "latency": 2183.013, "stderr": 8.212, "cost_per_test": 17.029068, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 23.333, "latency": 1323.029, "stderr": 7.854, "cost_per_test": 0.472427, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 23.295, "latency": 1182.24, "stderr": 7.841, "cost_per_test": 5.663294, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 20.0, "latency": 604.74, "stderr": 7.428, "cost_per_test": 2.510462, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 1204.529, "stderr": 7.428, "cost_per_test": 4.081424, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 2610.393, "stderr": 7.428, "cost_per_test": 1.651978, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 20.0, "latency": 3046.844, "stderr": 7.428, "cost_per_test": 8.653012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 19.0, "latency": 2947.783, "stderr": 7.117, "cost_per_test": 1.886037, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 16.667, "latency": 699.569, "stderr": 6.92, "cost_per_test": 1.075954, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 16.667, "latency": 4094.142, "stderr": 6.92, "cost_per_test": 5.443978, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 16.667, "latency": 4234.396, "stderr": 6.92, "cost_per_test": 11.036173, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 14.667, "latency": 679.661, "stderr": 6.191, "cost_per_test": 0.368144, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 13.333, "latency": 971.384, "stderr": 6.312, "cost_per_test": 7.426143, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 13.333, "latency": 1027.021, "stderr": 6.312, "cost_per_test": 5.960847, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 13.333, "latency": 1111.152, "stderr": 6.312, "cost_per_test": 2.079234, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 13.333, "latency": 1354.622, "stderr": 6.312, "cost_per_test": 0.601855, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 13.333, "latency": 1495.787, "stderr": 6.312, "cost_per_test": 0.147433, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 13.333, "latency": 3020.854, "stderr": 6.312, "cost_per_test": 3.745616, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 13.133, "latency": 562.149, "stderr": 6.22, "cost_per_test": 1.989157, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 12.333, "latency": 1303.848, "stderr": 5.905, "cost_per_test": 0.703688, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 10.333, "latency": 826.163, "stderr": 4.971, "cost_per_test": 1.13869, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4.5": { "accuracy": 10.0, "latency": 296.318, "stderr": 5.571, "cost_per_test": 0.566258, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 10.0, "latency": 815.165, "stderr": 5.571, "cost_per_test": 1.403597, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 10.0, "latency": 1096.628, "stderr": 5.571, "cost_per_test": 4.706236, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.0, "latency": 1181.176, "stderr": 5.571, "cost_per_test": 0.143783, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 10.0, "latency": 1200.277, "stderr": 5.571, "cost_per_test": 1.879269, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 10.0, "latency": 2124.134, "stderr": 5.571, "cost_per_test": 3.173128, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 10.0, "latency": 2724.08, "stderr": 5.571, "cost_per_test": 0.499628, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 9.746, "latency": 881.823, "stderr": 5.43, "cost_per_test": 0.376296, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 7.0, "latency": 142.434, "stderr": 3.9, "cost_per_test": 0.301074, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 6.667, "latency": 109.553, "stderr": 4.632, "cost_per_test": 0.524274, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 6.667, "latency": 207.327, "stderr": 4.632, "cost_per_test": 0.923283, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 6.667, "latency": 366.514, "stderr": 4.632, "cost_per_test": 0.030442, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 6.667, "latency": 408.237, "stderr": 4.632, "cost_per_test": 0.104081, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-sol": { "accuracy": 6.667, "latency": 1129.044, "stderr": 4.632, "cost_per_test": 2.192608, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 6.667, "latency": 1316.356, "stderr": 4.632, "cost_per_test": 1.303605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 6.667, "latency": 1372.497, "stderr": 4.632, "cost_per_test": 0.832712, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 6.667, "latency": 2538.969, "stderr": 4.632, "cost_per_test": 0.235158, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 6.667, "latency": 2580.136, "stderr": 4.632, "cost_per_test": 0.629331, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 6.667, "latency": 3496.18, "stderr": 4.632, "cost_per_test": 1.356768, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 6.524, "latency": 366.09, "stderr": 4.534, "cost_per_test": 0.172383, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 5.667, "latency": 1814.296, "stderr": 4.002, "cost_per_test": 0.816262, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 3.333, "latency": 59.71, "stderr": 3.333, "cost_per_test": 0.092032, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 3.333, "latency": 111.652, "stderr": 3.333, "cost_per_test": 0.100435, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 3.333, "latency": 125.067, "stderr": 3.333, "cost_per_test": 0.358641, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 3.333, "latency": 338.034, "stderr": 3.333, "cost_per_test": 1.530941, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 3.333, "latency": 342.732, "stderr": 3.333, "cost_per_test": 0.101653, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 3.333, "latency": 352.585, "stderr": 3.333, "cost_per_test": 1.210646, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 3.333, "latency": 365.534, "stderr": 3.333, "cost_per_test": 0.169527, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 3.333, "latency": 394.763, "stderr": 3.333, "cost_per_test": 0.316092, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 3.333, "latency": 407.264, "stderr": 3.333, "cost_per_test": 0.145904, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 3.333, "latency": 424.4, "stderr": 3.333, "cost_per_test": 0.043785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 3.333, "latency": 1040.508, "stderr": 3.333, "cost_per_test": 0.568809, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 3.333, "latency": 1050.002, "stderr": 3.333, "cost_per_test": 0.054776, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 3.333, "latency": 1056.728, "stderr": 3.333, "cost_per_test": 0.812549, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 3.333, "latency": 3907.926, "stderr": 3.333, "cost_per_test": 3.413393, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 3.083, "latency": 267.542, "stderr": 3.083, "cost_per_test": 0.960272, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 2.333, "latency": 255.015, "stderr": 2.333, "cost_per_test": 0.526926, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 2.333, "latency": 277.282, "stderr": 2.333, "cost_per_test": 0.077802, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 2.333, "latency": 295.62, "stderr": 2.333, "cost_per_test": 0.180353, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 22.448, "stderr": 0.0, "cost_per_test": 0.014665, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 37.426, "stderr": 0.0, "cost_per_test": 0.02667, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 103.548, "stderr": 0.0, "cost_per_test": 0.122994, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 117.479, "stderr": 0.0, "cost_per_test": 0.155222, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 140.944, "stderr": 0.0, "cost_per_test": 0.341242, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 174.648, "stderr": 0.0, "cost_per_test": 0.261882, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 564.048, "stderr": 0.0, "cost_per_test": 0.02036, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2038.781, "stderr": 0.0, "cost_per_test": 0.045743, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass": { "anthropic/claude-fable-5-1": { "accuracy": 77.642, "latency": 1714.017, "stderr": 2.835, "cost_per_test": 13.180073, "token_totals": { "input_tokens": 449166918, "output_tokens": 28237610, "reasoning_tokens": 7224, "cache_read_tokens": 365930262, "cache_write_tokens": 83209817 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 76.231, "latency": 985.634, "stderr": 2.874, "cost_per_test": 3.473645, "token_totals": { "input_tokens": 285519334, "output_tokens": 14493886, "reasoning_tokens": 6146392, "cache_read_tokens": 200405700, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 75.06, "latency": 1008.306, "stderr": 2.956, "cost_per_test": 5.136554, "token_totals": { "input_tokens": 322077193, "output_tokens": 16761490, "reasoning_tokens": 45865, "cache_read_tokens": 250544336, "cache_write_tokens": 71504899 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 75.001, "latency": 1084.296, "stderr": 2.915, "cost_per_test": 2.147235, "token_totals": { "input_tokens": 257606995, "output_tokens": 9968530, "reasoning_tokens": 6086406, "cache_read_tokens": 214574250, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 74.908, "latency": 1264.434, "stderr": 2.917, "cost_per_test": 0.459454, "token_totals": { "input_tokens": 779160448, "output_tokens": 53561725, "reasoning_tokens": 45716077, "cache_read_tokens": 712066688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 73.395, "latency": 3560.542, "stderr": 2.981, "cost_per_test": 9.245466, "token_totals": { "input_tokens": 1150865417, "output_tokens": 108029684, "reasoning_tokens": 94199819, "cache_read_tokens": 954147139, "cache_write_tokens": 196666034 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 73.089, "latency": 3052.932, "stderr": 2.973, "cost_per_test": 1.80097, "token_totals": { "input_tokens": 229302538, "output_tokens": 47417212, "reasoning_tokens": 1131, "cache_read_tokens": 154199490, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 72.441, "latency": 248.818, "stderr": 2.879, "cost_per_test": 0.389917, "token_totals": { "input_tokens": 142091226, "output_tokens": 6095824, "reasoning_tokens": 3452149, "cache_read_tokens": 116606720, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 71.934, "latency": 505.365, "stderr": 2.917, "cost_per_test": 0.319558, "token_totals": { "input_tokens": 95475291, "output_tokens": 4737687, "reasoning_tokens": 2609786, "cache_read_tokens": 70731373, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 71.161, "latency": 492.843, "stderr": 3.023, "cost_per_test": 1.884439, "token_totals": { "input_tokens": 225535207, "output_tokens": 8770631, "reasoning_tokens": 5466021, "cache_read_tokens": 187857044, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 70.788, "latency": 657.844, "stderr": 3.005, "cost_per_test": 0.978328, "token_totals": { "input_tokens": 139046016, "output_tokens": 7673212, "reasoning_tokens": 4482527, "cache_read_tokens": 90209280, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.499, "latency": 4012.548, "stderr": 3.145, "cost_per_test": 15.086184, "token_totals": { "input_tokens": 815304119, "output_tokens": 88000495, "reasoning_tokens": 76703031, "cache_read_tokens": 648981098, "cache_write_tokens": 166280701 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 69.822, "latency": 866.628, "stderr": 3.118, "cost_per_test": 6.696087, "token_totals": { "input_tokens": 104187949, "output_tokens": 14106474, "reasoning_tokens": 10318710, "cache_read_tokens": 62150339, "cache_write_tokens": 42027339 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 68.672, "latency": 2415.159, "stderr": 3.083, "cost_per_test": 2.790127, "token_totals": { "input_tokens": 444117224, "output_tokens": 14995598, "reasoning_tokens": 10717157, "cache_read_tokens": 381463574, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/ember-1": { "accuracy": 68.044, "latency": 1845.939, "stderr": 3.128, "cost_per_test": 2.972904, "token_totals": { "input_tokens": 638768422, "output_tokens": 13714742, "reasoning_tokens": 9058386, "cache_read_tokens": 574529181, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.955, "latency": 2681.175, "stderr": 3.123, "cost_per_test": 7.513965, "token_totals": { "input_tokens": 608937344, "output_tokens": 36375165, "reasoning_tokens": 31251074, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 66.771, "latency": 178.329, "stderr": 3.148, "cost_per_test": 0.857214, "token_totals": { "input_tokens": 262586790, "output_tokens": 7012701, "reasoning_tokens": 4789793, "cache_read_tokens": 208172664, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.96, "latency": 2436.106, "stderr": 3.181, "cost_per_test": 1.552558, "token_totals": { "input_tokens": 201221502, "output_tokens": 20553651, "reasoning_tokens": 16744468, "cache_read_tokens": 130986496, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 65.596, "latency": 996.469, "stderr": 3.238, "cost_per_test": 1.796539, "token_totals": { "input_tokens": 280409764, "output_tokens": 10821528, "reasoning_tokens": 6708987, "cache_read_tokens": 186011136, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 65.199, "latency": 3630.72, "stderr": 3.188, "cost_per_test": 4.829225, "token_totals": { "input_tokens": 608154133, "output_tokens": 46971256, "reasoning_tokens": 41166217, "cache_read_tokens": 508089082, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 64.939, "latency": 1547.695, "stderr": 3.214, "cost_per_test": 0.134306, "token_totals": { "input_tokens": 136588210, "output_tokens": 12521886, "reasoning_tokens": 8284595, "cache_read_tokens": 102894848, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 64.906, "latency": 479.2, "stderr": 3.174, "cost_per_test": 1.68964, "token_totals": { "input_tokens": 58571818, "output_tokens": 7476677, "reasoning_tokens": 5120560, "cache_read_tokens": 39455480, "cache_write_tokens": 19099407 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 64.854, "latency": 707.307, "stderr": 3.11, "cost_per_test": 1.193501, "token_totals": { "input_tokens": 116241036, "output_tokens": 7423691, "reasoning_tokens": 4447812, "cache_read_tokens": 91848017, "cache_write_tokens": 24342491 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 64.323, "latency": 283.566, "stderr": 3.156, "cost_per_test": 0.473604, "token_totals": { "input_tokens": 84693340, "output_tokens": 3280682, "reasoning_tokens": 949587, "cache_read_tokens": 57450112, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 63.864, "latency": 279.035, "stderr": 3.137, "cost_per_test": 1.274806, "token_totals": { "input_tokens": 82936206, "output_tokens": 4070683, "reasoning_tokens": 1866988, "cache_read_tokens": 65036497, "cache_write_tokens": 17803026 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 63.712, "latency": 2464.028, "stderr": 3.26, "cost_per_test": 0.585277, "token_totals": { "input_tokens": 181603577, "output_tokens": 28903691, "reasoning_tokens": 24008954, "cache_read_tokens": 139882560, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 63.335, "latency": 900.586, "stderr": 3.147, "cost_per_test": 5.7963, "token_totals": { "input_tokens": 115812555, "output_tokens": 12032734, "reasoning_tokens": 9165100, "cache_read_tokens": 71528751, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 63.294, "latency": 3438.427, "stderr": 3.229, "cost_per_test": 2.72124, "token_totals": { "input_tokens": 561529891, "output_tokens": 26978904, "reasoning_tokens": null, "cache_read_tokens": 287912448, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 62.563, "latency": 337.343, "stderr": 3.136, "cost_per_test": 0.156388, "token_totals": { "input_tokens": 118557354, "output_tokens": 14769391, "reasoning_tokens": 11893686, "cache_read_tokens": 97366102, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 62.458, "latency": 349.076, "stderr": 3.155, "cost_per_test": 0.134662, "token_totals": { "input_tokens": 114496920, "output_tokens": 15566679, "reasoning_tokens": 12729723, "cache_read_tokens": 91970688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 62.31, "latency": 1213.878, "stderr": 3.2, "cost_per_test": 1.323601, "token_totals": { "input_tokens": 149308416, "output_tokens": 12717430, "reasoning_tokens": 9488118, "cache_read_tokens": 103439338, "cache_write_tokens": 45805894 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 62.273, "latency": 1158.894, "stderr": 3.186, "cost_per_test": 1.794774, "token_totals": { "input_tokens": 187222743, "output_tokens": 21888011, "reasoning_tokens": 10753790, "cache_read_tokens": 148056139, "cache_write_tokens": 39133536 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 62.203, "latency": 2973.647, "stderr": 3.222, "cost_per_test": 1.164207, "token_totals": { "input_tokens": 782193467, "output_tokens": 143162719, "reasoning_tokens": 133330984, "cache_read_tokens": 613966554, "cache_write_tokens": 168173092 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 61.956, "latency": 1297.006, "stderr": 3.256, "cost_per_test": 0.138062, "token_totals": { "input_tokens": 194143568, "output_tokens": 28577937, "reasoning_tokens": 24325230, "cache_read_tokens": 139704320, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 60.815, "latency": 2565.695, "stderr": 3.263, "cost_per_test": 0.465359, "token_totals": { "input_tokens": 585632572, "output_tokens": 50154883, "reasoning_tokens": 43456905, "cache_read_tokens": 483082516, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 60.456, "latency": 951.811, "stderr": 3.216, "cost_per_test": 4.073277, "token_totals": { "input_tokens": 350470189, "output_tokens": 12925648, "reasoning_tokens": 10017673, "cache_read_tokens": 301365504, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 60.038, "latency": 1333.825, "stderr": 3.28, "cost_per_test": 0.81533, "token_totals": { "input_tokens": 380333007, "output_tokens": 36121135, "reasoning_tokens": 30086863, "cache_read_tokens": 313713664, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 59.904, "latency": 1038.928, "stderr": 3.13, "cost_per_test": 0.048982, "token_totals": { "input_tokens": 168296091, "output_tokens": 12132305, "reasoning_tokens": 7352234, "cache_read_tokens": 127944128, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 59.539, "latency": 1288.515, "stderr": 3.09, "cost_per_test": 0.55338, "token_totals": { "input_tokens": 311523716, "output_tokens": 16401077, "reasoning_tokens": 0, "cache_read_tokens": 262109696, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 58.864, "latency": 1995.736, "stderr": 3.211, "cost_per_test": 0.190956, "token_totals": { "input_tokens": 449103207, "output_tokens": 40664272, "reasoning_tokens": 37199155, "cache_read_tokens": 368052701, "cache_write_tokens": 80918209 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 58.663, "latency": 1596.755, "stderr": 3.138, "cost_per_test": 0.729136, "token_totals": { "input_tokens": 172298347, "output_tokens": 21059828, "reasoning_tokens": 17608474, "cache_read_tokens": 133313152, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 58.499, "latency": 1244.635, "stderr": 3.132, "cost_per_test": 0.719152, "token_totals": { "input_tokens": 171099768, "output_tokens": 21311454, "reasoning_tokens": 17772506, "cache_read_tokens": 134364160, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 57.665, "latency": 87.24, "stderr": 3.313, "cost_per_test": 0.483978, "token_totals": { "input_tokens": 104490097, "output_tokens": 3925073, "reasoning_tokens": 2390046, "cache_read_tokens": 68715157, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 56.86, "latency": 119.905, "stderr": 2.318, "cost_per_test": 0.269212, "token_totals": { "input_tokens": 84100295, "output_tokens": 3814259, "reasoning_tokens": 1704398, "cache_read_tokens": 63070995, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 53.526, "latency": 233.001, "stderr": 3.302, "cost_per_test": 0.859679, "token_totals": { "input_tokens": 335113206, "output_tokens": 5873488, "reasoning_tokens": 4405887, "cache_read_tokens": 288602367, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 53.045, "latency": 1052.036, "stderr": 3.29, "cost_per_test": 2.277613, "token_totals": { "input_tokens": 386182808, "output_tokens": 18761287, "reasoning_tokens": 15767772, "cache_read_tokens": 316266542, "cache_write_tokens": 69503073 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 52.066, "latency": 701.825, "stderr": 2.387, "cost_per_test": 0.883559, "token_totals": { "input_tokens": 413690950, "output_tokens": 3571385, "reasoning_tokens": 1667757, "cache_read_tokens": 329680688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 49.895, "latency": 105.32, "stderr": 3.302, "cost_per_test": 0.356293, "token_totals": { "input_tokens": 67586371, "output_tokens": 3157887, "reasoning_tokens": 2191235, "cache_read_tokens": 41703106, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 49.691, "latency": 314.575, "stderr": 3.248, "cost_per_test": 0.164063, "token_totals": { "input_tokens": 85169988, "output_tokens": 4820320, "reasoning_tokens": 176867, "cache_read_tokens": 22014292, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 47.407, "latency": 209.726, "stderr": 3.237, "cost_per_test": 0.435494, "token_totals": { "input_tokens": 66735457, "output_tokens": 3616925, "reasoning_tokens": 2247001, "cache_read_tokens": 48851200, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 45.697, "latency": 333.833, "stderr": 3.318, "cost_per_test": 0.025079, "token_totals": { "input_tokens": 142160419, "output_tokens": 8349782, "reasoning_tokens": null, "cache_read_tokens": 108172032, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 45.135, "latency": 924.425, "stderr": 3.225, "cost_per_test": 0.565511, "token_totals": { "input_tokens": 247358580, "output_tokens": 8695317, "reasoning_tokens": 6542577, "cache_read_tokens": 203326968, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 43.516, "latency": 810.557, "stderr": 2.947, "cost_per_test": 0.353757, "token_totals": { "input_tokens": 147459631, "output_tokens": 7269937, "reasoning_tokens": null, "cache_read_tokens": 130876928, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 41.461, "latency": 385.252, "stderr": 2.871, "cost_per_test": 0.091767, "token_totals": { "input_tokens": 107926904, "output_tokens": 7430357, "reasoning_tokens": null, "cache_read_tokens": 98264704, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 38.713, "latency": 310.263, "stderr": 2.81, "cost_per_test": 0.093885, "token_totals": { "input_tokens": 83119598, "output_tokens": 4291556, "reasoning_tokens": 2832859, "cache_read_tokens": 61095936, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 37.842, "latency": 100.956, "stderr": 3.098, "cost_per_test": 0.301555, "token_totals": { "input_tokens": 33380072, "output_tokens": 1903235, "reasoning_tokens": 1460348, "cache_read_tokens": 17443775, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 37.557, "latency": 100.623, "stderr": 3.13, "cost_per_test": 0.100699, "token_totals": { "input_tokens": 39222064, "output_tokens": 3147091, "reasoning_tokens": 2378293, "cache_read_tokens": 21372192, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 37.451, "latency": 47.461, "stderr": 3.115, "cost_per_test": 0.081536, "token_totals": { "input_tokens": 74874616, "output_tokens": 1868593, "reasoning_tokens": 1185933, "cache_read_tokens": 42212661, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 36.586, "latency": 1006.185, "stderr": 3.095, "cost_per_test": 0.806171, "token_totals": { "input_tokens": 201253946, "output_tokens": 26480732, "reasoning_tokens": 25245242, "cache_read_tokens": 169648384, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 34.787, "latency": 376.514, "stderr": 2.621, "cost_per_test": 0.040042, "token_totals": { "input_tokens": 66417509, "output_tokens": 2436638, "reasoning_tokens": 1177368, "cache_read_tokens": 53971712, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.3": { "accuracy": 34.416, "latency": 97.519, "stderr": 2.994, "cost_per_test": 0.116616, "token_totals": { "input_tokens": 37643889, "output_tokens": 1781754, "reasoning_tokens": 1092969, "cache_read_tokens": 27621238, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 32.844, "latency": 378.159, "stderr": 2.752, "cost_per_test": 0.298017, "token_totals": { "input_tokens": 94733985, "output_tokens": 3383446, "reasoning_tokens": 2053059, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 31.241, "latency": 89.262, "stderr": 2.704, "cost_per_test": 0.119262, "token_totals": { "input_tokens": 19969068, "output_tokens": 1606938, "reasoning_tokens": 960969, "cache_read_tokens": 14756820, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 29.163, "latency": 500.826, "stderr": 2.713, "cost_per_test": 0.017023, "token_totals": { "input_tokens": 114032276, "output_tokens": 3002594, "reasoning_tokens": 1387062, "cache_read_tokens": 98540288, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 28.314, "latency": 155.016, "stderr": 2.552, "cost_per_test": 0.226416, "token_totals": { "input_tokens": 73419285, "output_tokens": 2773686, "reasoning_tokens": 1184967, "cache_read_tokens": 53705606, "cache_write_tokens": 18982650 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 27.291, "latency": 430.673, "stderr": 2.824, "cost_per_test": 1.443655, "token_totals": { "input_tokens": 150690955, "output_tokens": 7011852, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 26.578, "latency": 220.306, "stderr": 2.518, "cost_per_test": 0.069662, "token_totals": { "input_tokens": 154740627, "output_tokens": 4455634, "reasoning_tokens": 3038667, "cache_read_tokens": 128182784, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 25.382, "latency": 302.854, "stderr": 2.59, "cost_per_test": 0.120058, "token_totals": { "input_tokens": 74679791, "output_tokens": 2724867, "reasoning_tokens": 1450884, "cache_read_tokens": 10592268, "cache_write_tokens": 4764496 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 20.085, "latency": 18.744, "stderr": 2.457, "cost_per_test": 0.014308, "token_totals": { "input_tokens": 6757967, "output_tokens": 906826, "reasoning_tokens": 487639, "cache_read_tokens": 1281568, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 13.002, "latency": 1180.079, "stderr": 2.049, "cost_per_test": 0.030533, "token_totals": { "input_tokens": 69555925, "output_tokens": 22469385, "reasoning_tokens": 14413630, "cache_read_tokens": 51967923, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 12.41, "latency": 29.67, "stderr": 1.805, "cost_per_test": 0.023633, "token_totals": { "input_tokens": 43610163, "output_tokens": 1545275, "reasoning_tokens": 1248847, "cache_read_tokens": 29554557, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "weighted_pass__fact_pattern_analysis": { "anthropic/claude-fable-5-1": { "accuracy": 79.29, "latency": 1615.462, "stderr": 5.876, "cost_per_test": 11.625261, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 77.336, "latency": 962.376, "stderr": 6.157, "cost_per_test": 4.479733, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 76.678, "latency": 812.519, "stderr": 6.107, "cost_per_test": 2.8502, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 75.079, "latency": 626.149, "stderr": 6.087, "cost_per_test": 0.340841, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 74.764, "latency": 1141.924, "stderr": 6.372, "cost_per_test": 2.157843, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 74.261, "latency": 879.948, "stderr": 6.329, "cost_per_test": 6.080646, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 73.852, "latency": 3212.209, "stderr": 6.306, "cost_per_test": 7.966557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 72.908, "latency": 3840.498, "stderr": 6.607, "cost_per_test": 14.475041, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 72.524, "latency": 1324.466, "stderr": 6.621, "cost_per_test": 0.460654, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 71.843, "latency": 2074.148, "stderr": 6.528, "cost_per_test": 3.85373, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 70.171, "latency": 260.736, "stderr": 6.186, "cost_per_test": 0.392251, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 67.954, "latency": 169.217, "stderr": 6.603, "cost_per_test": 0.792812, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 67.563, "latency": 504.76, "stderr": 6.892, "cost_per_test": 1.775974, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 66.015, "latency": 838.059, "stderr": 6.75, "cost_per_test": 5.184223, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 66.014, "latency": 1258.298, "stderr": 6.509, "cost_per_test": 0.050237, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.6": { "accuracy": 65.96, "latency": 721.879, "stderr": 6.737, "cost_per_test": 0.967508, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 65.371, "latency": 518.318, "stderr": 6.699, "cost_per_test": 1.616173, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 64.276, "latency": 946.959, "stderr": 6.947, "cost_per_test": 1.653949, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 63.25, "latency": 3662.48, "stderr": 7.161, "cost_per_test": 4.593033, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 62.2, "latency": 2637.671, "stderr": 7.06, "cost_per_test": 3.173861, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 62.106, "latency": 2653.408, "stderr": 7.048, "cost_per_test": 1.716952, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.713, "latency": 1028.913, "stderr": 7.024, "cost_per_test": 1.20312, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 61.632, "latency": 337.343, "stderr": 6.702, "cost_per_test": 1.387812, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 60.624, "latency": 408.316, "stderr": 6.913, "cost_per_test": 0.136181, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 60.429, "latency": 2784.942, "stderr": 7.197, "cost_per_test": 7.969384, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 60.23, "latency": 2051.965, "stderr": 6.901, "cost_per_test": 0.212665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 59.929, "latency": 1273.988, "stderr": 6.877, "cost_per_test": 1.783178, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 59.691, "latency": 1686.83, "stderr": 6.616, "cost_per_test": 0.629486, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 59.613, "latency": 3047.67, "stderr": 7.107, "cost_per_test": 0.549062, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 59.491, "latency": 309.359, "stderr": 7.095, "cost_per_test": 0.394055, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 59.228, "latency": 2681.727, "stderr": 6.866, "cost_per_test": 1.15627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 59.225, "latency": 956.476, "stderr": 7.076, "cost_per_test": 3.881333, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 58.414, "latency": 1869.987, "stderr": 7.311, "cost_per_test": 0.133577, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 58.245, "latency": 3230.755, "stderr": 7.294, "cost_per_test": 1.851233, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 58.095, "latency": 237.175, "stderr": 6.739, "cost_per_test": 0.753889, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 57.73, "latency": 1377.571, "stderr": 7.265, "cost_per_test": 0.834749, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 57.286, "latency": 1664.779, "stderr": 7.201, "cost_per_test": 0.163393, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 57.242, "latency": 4183.973, "stderr": 7.066, "cost_per_test": 4.077226, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 56.703, "latency": 845.013, "stderr": 7.131, "cost_per_test": 1.284935, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 56.368, "latency": 1302.263, "stderr": 7.173, "cost_per_test": 0.530547, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "tencent/hy4-preview": { "accuracy": 55.791, "latency": 2393.442, "stderr": 7.33, "cost_per_test": 0.531712, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 55.579, "latency": 1278.557, "stderr": 6.509, "cost_per_test": 0.651612, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 54.742, "latency": 348.287, "stderr": 6.993, "cost_per_test": 0.147096, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 52.476, "latency": 125.373, "stderr": 4.819, "cost_per_test": 0.261785, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 49.931, "latency": 939.613, "stderr": 7.233, "cost_per_test": 2.094899, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 49.834, "latency": 110.876, "stderr": 6.974, "cost_per_test": 0.342251, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 49.293, "latency": 87.896, "stderr": 7.176, "cost_per_test": 0.427212, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 46.747, "latency": 980.497, "stderr": 6.819, "cost_per_test": 0.607497, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 44.094, "latency": 954.507, "stderr": 6.562, "cost_per_test": 0.42429, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 43.85, "latency": 495.424, "stderr": 6.524, "cost_per_test": 0.104177, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-spark-open": { "accuracy": 43.596, "latency": 995.717, "stderr": 5.271, "cost_per_test": 2.528872, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 42.198, "latency": 224.965, "stderr": 6.834, "cost_per_test": 0.419675, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 38.825, "latency": 110.173, "stderr": 6.637, "cost_per_test": 0.093878, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 36.731, "latency": 96.392, "stderr": 6.482, "cost_per_test": 0.092552, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 36.518, "latency": 874.453, "stderr": 6.486, "cost_per_test": 0.736955, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 33.802, "latency": 368.229, "stderr": 6.56, "cost_per_test": 0.166429, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 33.026, "latency": 386.935, "stderr": 6.722, "cost_per_test": 0.025975, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 32.873, "latency": 115.164, "stderr": 6.49, "cost_per_test": 0.274484, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 32.253, "latency": 60.341, "stderr": 6.329, "cost_per_test": 0.086864, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 31.978, "latency": 455.125, "stderr": 5.535, "cost_per_test": 0.364885, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 29.256, "latency": 641.995, "stderr": 5.841, "cost_per_test": 0.01803, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 28.136, "latency": 356.11, "stderr": 5.659, "cost_per_test": 0.09907, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 25.051, "latency": 89.445, "stderr": 5.048, "cost_per_test": 0.120294, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 24.361, "latency": 532.451, "stderr": 5.832, "cost_per_test": 1.882853, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 23.369, "latency": 221.457, "stderr": 5.182, "cost_per_test": 0.070223, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 23.224, "latency": 426.904, "stderr": 5.057, "cost_per_test": 0.037953, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 22.853, "latency": 178.603, "stderr": 5.385, "cost_per_test": 0.219905, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 21.731, "latency": 23.467, "stderr": 5.405, "cost_per_test": 0.015379, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 20.219, "latency": 314.548, "stderr": 5.195, "cost_per_test": 0.114892, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "inception/mercury-2.5": { "accuracy": 11.667, "latency": 34.135, "stderr": 3.775, "cost_per_test": 0.024313, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 6.35, "latency": 1584.808, "stderr": 2.727, "cost_per_test": 0.034737, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__rule_source_lookup": { "zai/glm-5.3": { "accuracy": 81.291, "latency": 2863.402, "stderr": 6.797, "cost_per_test": 1.698419, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 77.978, "latency": 950.785, "stderr": 7.278, "cost_per_test": 5.207646, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 77.359, "latency": 818.772, "stderr": 7.249, "cost_per_test": 3.044362, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 74.684, "latency": 195.725, "stderr": 7.742, "cost_per_test": 0.307916, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 72.942, "latency": 1590.623, "stderr": 8.174, "cost_per_test": 11.284536, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 70.819, "latency": 1361.742, "stderr": 7.976, "cost_per_test": 0.492773, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 70.181, "latency": 339.091, "stderr": 7.405, "cost_per_test": 0.163641, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 68.105, "latency": 972.116, "stderr": 8.313, "cost_per_test": 1.973979, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 67.97, "latency": 1257.166, "stderr": 8.31, "cost_per_test": 0.750516, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 67.821, "latency": 3858.029, "stderr": 8.279, "cost_per_test": 9.831783, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 67.667, "latency": 413.788, "stderr": 8.261, "cost_per_test": 0.261741, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 67.655, "latency": 446.798, "stderr": 8.257, "cost_per_test": 1.712984, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 67.489, "latency": 489.476, "stderr": 8.255, "cost_per_test": 0.812812, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 66.389, "latency": 2822.295, "stderr": 8.217, "cost_per_test": 1.085204, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 65.916, "latency": 419.046, "stderr": 8.127, "cost_per_test": 0.728366, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 61.522, "latency": 1043.928, "stderr": 8.713, "cost_per_test": 1.678605, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 60.99, "latency": 151.907, "stderr": 8.652, "cost_per_test": 0.788644, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 60.848, "latency": 2255.415, "stderr": 8.629, "cost_per_test": 1.26074, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 60.816, "latency": 2077.708, "stderr": 8.647, "cost_per_test": 1.497169, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "stepfun/step-5-preview": { "accuracy": 60.8, "latency": 1297.966, "stderr": 7.729, "cost_per_test": 0.552303, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "grok/grok-4.5": { "accuracy": 60.637, "latency": 306.772, "stderr": 8.607, "cost_per_test": 0.387411, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 60.287, "latency": 430.403, "stderr": 6.065, "cost_per_test": 0.193793, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 60.237, "latency": 162.06, "stderr": 8.582, "cost_per_test": 0.789933, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 60.234, "latency": 1094.178, "stderr": 8.594, "cost_per_test": 0.110782, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 59.833, "latency": 1383.783, "stderr": 8.545, "cost_per_test": 0.119899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 58.356, "latency": 1574.775, "stderr": 8.877, "cost_per_test": 1.540859, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 57.902, "latency": 116.279, "stderr": 8.815, "cost_per_test": 0.471874, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 57.887, "latency": 153.734, "stderr": 8.809, "cost_per_test": 0.109333, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "tencent/hy4-preview": { "accuracy": 57.514, "latency": 2586.868, "stderr": 8.768, "cost_per_test": 0.636639, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 56.89, "latency": 302.327, "stderr": 8.709, "cost_per_test": 0.112383, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 56.298, "latency": 851.616, "stderr": 8.642, "cost_per_test": 0.043707, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 56.163, "latency": 3532.864, "stderr": 9.126, "cost_per_test": 13.741037, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 55.936, "latency": 747.145, "stderr": 9.091, "cost_per_test": 5.813849, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 55.145, "latency": 103.014, "stderr": 6.474, "cost_per_test": 0.228706, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 55.079, "latency": 1210.271, "stderr": 8.513, "cost_per_test": 0.724403, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 54.706, "latency": 342.933, "stderr": 8.916, "cost_per_test": 1.290724, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 54.292, "latency": 1875.339, "stderr": 8.874, "cost_per_test": 0.341665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 53.787, "latency": 2236.425, "stderr": 8.794, "cost_per_test": 5.48599, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 53.763, "latency": 246.939, "stderr": 8.772, "cost_per_test": 0.021168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 53.288, "latency": 2904.931, "stderr": 8.768, "cost_per_test": 4.09629, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 52.194, "latency": 919.518, "stderr": 9.087, "cost_per_test": 1.617708, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 51.16, "latency": 833.412, "stderr": 8.908, "cost_per_test": 1.158281, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 49.839, "latency": 1369.988, "stderr": 8.761, "cost_per_test": 0.668801, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 49.361, "latency": 705.261, "stderr": 8.658, "cost_per_test": 4.662827, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 49.179, "latency": 945.988, "stderr": 8.621, "cost_per_test": 1.02288, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 47.518, "latency": 126.702, "stderr": 8.463, "cost_per_test": 0.286683, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 46.785, "latency": 227.911, "stderr": 8.739, "cost_per_test": 0.71482, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 45.904, "latency": 1517.938, "stderr": 8.604, "cost_per_test": 0.123429, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 44.233, "latency": 799.357, "stderr": 8.826, "cost_per_test": 3.524307, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 44.162, "latency": 199.19, "stderr": 7.537, "cost_per_test": 0.073258, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 43.581, "latency": 94.714, "stderr": 8.697, "cost_per_test": 0.315209, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 41.128, "latency": 165.068, "stderr": 6.881, "cost_per_test": 0.02233, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-sol": { "accuracy": 39.766, "latency": 923.305, "stderr": 8.502, "cost_per_test": 1.829993, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 34.001, "latency": 41.274, "stderr": 8.336, "cost_per_test": 0.060332, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 32.161, "latency": 273.171, "stderr": 6.689, "cost_per_test": 0.206147, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 32.085, "latency": 222.832, "stderr": 6.639, "cost_per_test": 0.067974, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 31.327, "latency": 391.772, "stderr": 7.417, "cost_per_test": 0.215865, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 30.448, "latency": 678.113, "stderr": 8.031, "cost_per_test": 0.417333, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 29.323, "latency": 77.219, "stderr": 7.934, "cost_per_test": 0.085031, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 28.261, "latency": 86.667, "stderr": 8.04, "cost_per_test": 0.246919, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 26.976, "latency": 126.008, "stderr": 6.95, "cost_per_test": 0.209741, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 26.513, "latency": 687.26, "stderr": 7.594, "cost_per_test": 0.560241, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 26.26, "latency": 422.308, "stderr": 7.509, "cost_per_test": 1.333021, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.3": { "accuracy": 26.126, "latency": 85.502, "stderr": 7.511, "cost_per_test": 0.097472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 25.175, "latency": 231.03, "stderr": 6.804, "cost_per_test": 0.092243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 24.795, "latency": 78.155, "stderr": 7.169, "cost_per_test": 0.102575, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 23.259, "latency": 149.816, "stderr": 6.285, "cost_per_test": 0.05178, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 21.557, "latency": 366.583, "stderr": 6.323, "cost_per_test": 0.014016, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 20.684, "latency": 18.573, "stderr": 6.521, "cost_per_test": 0.010765, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 14.685, "latency": 732.305, "stderr": 6.205, "cost_per_test": 0.020854, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 8.279, "latency": 29.325, "stderr": 3.981, "cost_per_test": 0.021089, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "weighted_pass__current_temporal_analysis": { "google/gemini-4-argon": { "accuracy": 84.637, "latency": 1606.495, "stderr": 6.232, "cost_per_test": 5.143482, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 81.205, "latency": 2244.633, "stderr": 6.8, "cost_per_test": 16.518066, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 80.827, "latency": 2562.167, "stderr": 6.769, "cost_per_test": 3.703546, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 78.312, "latency": 1369.127, "stderr": 7.299, "cost_per_test": 2.742275, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 78.164, "latency": 4008.358, "stderr": 7.28, "cost_per_test": 2.275751, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-wonder": { "accuracy": 77.655, "latency": 653.458, "stderr": 7.258, "cost_per_test": 2.232327, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 77.239, "latency": 4722.823, "stderr": 7.235, "cost_per_test": 11.848978, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 76.033, "latency": 584.037, "stderr": 7.144, "cost_per_test": 1.945154, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 75.003, "latency": 1540.94, "stderr": 7.702, "cost_per_test": 0.541685, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 74.53, "latency": 1363.832, "stderr": 7.667, "cost_per_test": 2.443799, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 74.38, "latency": 3691.615, "stderr": 7.664, "cost_per_test": 9.388335, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 73.88, "latency": 1758.869, "stderr": 7.649, "cost_per_test": 0.154282, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.6": { "accuracy": 73.425, "latency": 812.418, "stderr": 7.584, "cost_per_test": 1.209783, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 73.299, "latency": 5343.646, "stderr": 7.593, "cost_per_test": 19.635953, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 73.142, "latency": 660.075, "stderr": 7.614, "cost_per_test": 0.354536, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 72.932, "latency": 3066.144, "stderr": 7.583, "cost_per_test": 3.074284, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 71.882, "latency": 397.458, "stderr": 7.468, "cost_per_test": 0.145689, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 71.631, "latency": 1207.608, "stderr": 8.048, "cost_per_test": 6.007301, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 71.449, "latency": 3069.918, "stderr": 8.024, "cost_per_test": 1.903846, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 70.994, "latency": 1206.807, "stderr": 7.991, "cost_per_test": 9.205921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 70.566, "latency": 331.714, "stderr": 7.933, "cost_per_test": 0.601971, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 70.529, "latency": 446.097, "stderr": 7.934, "cost_per_test": 0.208159, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 69.561, "latency": 326.12, "stderr": 7.92, "cost_per_test": 0.455908, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 68.827, "latency": 112.732, "stderr": 7.825, "cost_per_test": 0.562793, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 68.195, "latency": 1352.021, "stderr": 7.736, "cost_per_test": 5.233081, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 67.902, "latency": 2180.108, "stderr": 7.767, "cost_per_test": 0.952634, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 66.471, "latency": 326.113, "stderr": 8.142, "cost_per_test": 1.386875, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 66.224, "latency": 1384.946, "stderr": 8.122, "cost_per_test": 9.278145, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.078, "latency": 910.516, "stderr": 8.101, "cost_per_test": 1.404458, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 65.741, "latency": 1727.127, "stderr": 8.102, "cost_per_test": 1.136219, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 64.751, "latency": 228.113, "stderr": 8.527, "cost_per_test": 1.013323, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 64.214, "latency": 4157.048, "stderr": 8.498, "cost_per_test": 2.23471, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 63.664, "latency": 3258.22, "stderr": 8.443, "cost_per_test": 0.762342, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 63.366, "latency": 1908.876, "stderr": 8.378, "cost_per_test": 2.13313, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 61.648, "latency": 279.898, "stderr": 8.243, "cost_per_test": 0.929565, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 61.439, "latency": 1328.168, "stderr": 8.269, "cost_per_test": 0.058099, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 61.439, "latency": 1494.556, "stderr": 8.708, "cost_per_test": 0.158431, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 60.331, "latency": 4946.592, "stderr": 8.57, "cost_per_test": 6.593928, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 60.069, "latency": 3317.224, "stderr": 8.547, "cost_per_test": 0.575904, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 59.973, "latency": 1517.93, "stderr": 8.549, "cost_per_test": 2.342482, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 58.681, "latency": 1734.264, "stderr": 7.947, "cost_per_test": 0.698028, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 58.573, "latency": 1947.331, "stderr": 8.894, "cost_per_test": 1.177468, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 56.197, "latency": 3939.143, "stderr": 8.712, "cost_per_test": 1.419874, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 56.104, "latency": 1553.84, "stderr": 8.566, "cost_per_test": 3.706383, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 55.344, "latency": 2764.929, "stderr": 8.476, "cost_per_test": 0.233793, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 55.238, "latency": 152.954, "stderr": 5.187, "cost_per_test": 0.290519, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 52.905, "latency": 477.413, "stderr": 8.65, "cost_per_test": 0.214205, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 51.239, "latency": 1213.636, "stderr": 8.484, "cost_per_test": 0.756993, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 51.028, "latency": 372.31, "stderr": 8.914, "cost_per_test": 0.027631, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 49.747, "latency": 303.119, "stderr": 8.688, "cost_per_test": 0.551759, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 48.604, "latency": 132.729, "stderr": 8.557, "cost_per_test": 0.396307, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 47.453, "latency": 502.508, "stderr": 6.978, "cost_per_test": 0.088018, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-spark-open": { "accuracy": 47.221, "latency": 760.495, "stderr": 5.768, "cost_per_test": 0.288385, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 46.103, "latency": 398.044, "stderr": 7.28, "cost_per_test": 0.105531, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 42.226, "latency": 451.743, "stderr": 7.189, "cost_per_test": 0.04028, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 38.553, "latency": 1413.578, "stderr": 8.274, "cost_per_test": 1.066462, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 37.683, "latency": 130.181, "stderr": 7.853, "cost_per_test": 0.160784, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 36.539, "latency": 138.829, "stderr": 7.942, "cost_per_test": 0.312752, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 34.603, "latency": 129.458, "stderr": 8.004, "cost_per_test": 0.13418, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 34.385, "latency": 916.798, "stderr": 6.99, "cost_per_test": 0.3676, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 31.86, "latency": 123.063, "stderr": 7.882, "cost_per_test": 0.099933, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 30.497, "latency": 472.355, "stderr": 7.743, "cost_per_test": 0.301399, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 30.085, "latency": 504.95, "stderr": 7.199, "cost_per_test": 0.013685, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 29.444, "latency": 420.093, "stderr": 7.828, "cost_per_test": 1.381649, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 25.45, "latency": 168.511, "stderr": 6.48, "cost_per_test": 0.238473, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 24.916, "latency": 52.71, "stderr": 7.269, "cost_per_test": 0.075425, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 22.52, "latency": 283.728, "stderr": 6.6, "cost_per_test": 0.083781, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 21.497, "latency": 413.595, "stderr": 6.301, "cost_per_test": 0.135098, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "inception/mercury-2.5": { "accuracy": 13.201, "latency": 36.086, "stderr": 4.973, "cost_per_test": 0.022182, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 12.31, "latency": 21.131, "stderr": 5.236, "cost_per_test": 0.013534, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 10.469, "latency": 903.722, "stderr": 5.136, "cost_per_test": 0.027891, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__numbers_calculations": { "anthropic/claude-opus-5-5": { "accuracy": 88.878, "latency": 2259.551, "stderr": 5.519, "cost_per_test": 7.944016, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 87.97, "latency": 752.921, "stderr": 5.488, "cost_per_test": 5.30143, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 87.616, "latency": 526.541, "stderr": 5.474, "cost_per_test": 1.946484, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 85.388, "latency": 2198.898, "stderr": 6.232, "cost_per_test": 5.159882, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 85.168, "latency": 502.859, "stderr": 6.222, "cost_per_test": 2.664386, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 84.524, "latency": 2268.254, "stderr": 6.185, "cost_per_test": 1.236253, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 83.973, "latency": 1148.921, "stderr": 5.49, "cost_per_test": 1.425651, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 83.382, "latency": 325.851, "stderr": 6.171, "cost_per_test": 2.303921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 83.051, "latency": 120.595, "stderr": 6.134, "cost_per_test": 0.619008, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 81.788, "latency": 343.387, "stderr": 6.098, "cost_per_test": 0.183382, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 81.627, "latency": 1742.081, "stderr": 6.053, "cost_per_test": 2.186392, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 81.388, "latency": 783.344, "stderr": 6.086, "cost_per_test": 2.145907, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 81.055, "latency": 128.464, "stderr": 6.104, "cost_per_test": 0.187818, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "tencent/hy4-preview": { "accuracy": 81.045, "latency": 1213.722, "stderr": 6.782, "cost_per_test": 0.2941, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 81.017, "latency": 710.489, "stderr": 6.14, "cost_per_test": 0.077152, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 80.451, "latency": 384.892, "stderr": 6.107, "cost_per_test": 0.609108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 79.76, "latency": 424.439, "stderr": 6.703, "cost_per_test": 0.595682, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 79.503, "latency": 677.794, "stderr": 6.7, "cost_per_test": 0.241521, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 78.846, "latency": 70.814, "stderr": 6.614, "cost_per_test": 0.355132, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 78.629, "latency": 250.763, "stderr": 6.686, "cost_per_test": 0.800257, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 78.381, "latency": 2328.746, "stderr": 6.624, "cost_per_test": 1.946576, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 78.048, "latency": 178.13, "stderr": 6.116, "cost_per_test": 0.071997, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 77.948, "latency": 94.184, "stderr": 6.587, "cost_per_test": 0.306179, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 77.876, "latency": 258.28, "stderr": 5.968, "cost_per_test": 1.623587, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 77.664, "latency": 752.627, "stderr": 6.655, "cost_per_test": 1.027503, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 77.344, "latency": 387.008, "stderr": 6.578, "cost_per_test": 0.381672, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 76.982, "latency": 142.817, "stderr": 6.655, "cost_per_test": 0.550501, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 76.78, "latency": 1793.902, "stderr": 7.246, "cost_per_test": 0.539077, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.764, "latency": 333.476, "stderr": 6.509, "cost_per_test": 1.539682, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 76.696, "latency": 540.429, "stderr": 7.179, "cost_per_test": 0.829064, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 76.568, "latency": 150.488, "stderr": 6.643, "cost_per_test": 0.051675, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 76.295, "latency": 73.558, "stderr": 5.364, "cost_per_test": 0.149096, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 76.278, "latency": 455.508, "stderr": 7.154, "cost_per_test": 0.876776, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 75.563, "latency": 946.298, "stderr": 7.112, "cost_per_test": 0.546571, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 75.514, "latency": 784.284, "stderr": 7.145, "cost_per_test": 0.063015, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder": { "accuracy": 74.438, "latency": 206.578, "stderr": 7.066, "cost_per_test": 0.792122, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-luna": { "accuracy": 73.764, "latency": 819.844, "stderr": 7.005, "cost_per_test": 0.070109, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 72.498, "latency": 234.86, "stderr": 7.561, "cost_per_test": 0.855352, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 72.496, "latency": 148.479, "stderr": 7.044, "cost_per_test": 0.257231, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 72.083, "latency": 647.942, "stderr": 6.942, "cost_per_test": 0.235703, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 71.752, "latency": 1086.03, "stderr": 7.459, "cost_per_test": 0.196728, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 70.583, "latency": 504.122, "stderr": 7.014, "cost_per_test": 0.729707, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 70.183, "latency": 726.33, "stderr": 6.951, "cost_per_test": 0.320631, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 69.533, "latency": 44.318, "stderr": 6.806, "cost_per_test": 0.059752, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 69.483, "latency": 384.955, "stderr": 7.333, "cost_per_test": 0.735903, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 68.911, "latency": 339.074, "stderr": 5.042, "cost_per_test": 0.15679, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 67.123, "latency": 88.311, "stderr": 7.128, "cost_per_test": 0.082771, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 66.567, "latency": 185.832, "stderr": 7.606, "cost_per_test": 0.012388, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 66.459, "latency": 101.165, "stderr": 7.168, "cost_per_test": 0.206993, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 66.225, "latency": 687.242, "stderr": 7.652, "cost_per_test": 0.406577, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 62.87, "latency": 455.905, "stderr": 7.399, "cost_per_test": 0.260482, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 62.435, "latency": 564.446, "stderr": 7.312, "cost_per_test": 0.024687, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling": { "accuracy": 61.901, "latency": 466.325, "stderr": 6.478, "cost_per_test": 0.214027, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 61.752, "latency": 542.04, "stderr": 7.741, "cost_per_test": 0.24358, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 60.565, "latency": 127.478, "stderr": 7.763, "cost_per_test": 0.245076, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 54.9, "latency": 196.912, "stderr": 8.072, "cost_per_test": 0.085401, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 52.296, "latency": 551.988, "stderr": 8.154, "cost_per_test": 0.395054, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 52.064, "latency": 198.363, "stderr": 5.258, "cost_per_test": 0.046584, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 51.357, "latency": 56.426, "stderr": 7.635, "cost_per_test": 0.056398, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 51.226, "latency": 269.336, "stderr": 7.238, "cost_per_test": 0.062924, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 50.407, "latency": 94.288, "stderr": 6.141, "cost_per_test": 0.105905, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 49.085, "latency": 180.956, "stderr": 6.891, "cost_per_test": 0.128621, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 48.818, "latency": 52.646, "stderr": 6.934, "cost_per_test": 0.060833, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 47.179, "latency": 178.17, "stderr": 7.297, "cost_per_test": 0.072386, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 45.318, "latency": 224.875, "stderr": 7.623, "cost_per_test": 0.68906, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 43.247, "latency": 311.354, "stderr": 6.762, "cost_per_test": 0.009895, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 43.12, "latency": 287.096, "stderr": 5.923, "cost_per_test": 0.021077, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 42.802, "latency": 116.279, "stderr": 6.49, "cost_per_test": 0.028897, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 39.5, "latency": 20.929, "stderr": 7.717, "cost_per_test": 0.013865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 31.499, "latency": 978.65, "stderr": 6.731, "cost_per_test": 0.024076, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 31.134, "latency": 22.004, "stderr": 5.841, "cost_per_test": 0.015262, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "weighted_pass__forms_filings": { "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 79.399, "latency": 1400.675, "stderr": 6.669, "cost_per_test": 0.547143, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 76.552, "latency": 1264.709, "stderr": 7.157, "cost_per_test": 2.488422, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 73.177, "latency": 2185.691, "stderr": 7.553, "cost_per_test": 17.995831, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 72.508, "latency": 3672.354, "stderr": 7.509, "cost_per_test": 11.447471, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 72.081, "latency": 800.747, "stderr": 7.457, "cost_per_test": 1.21292, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 71.707, "latency": 635.944, "stderr": 6.899, "cost_per_test": 0.399484, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 71.666, "latency": 3013.858, "stderr": 7.423, "cost_per_test": 1.836345, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 71.661, "latency": 3738.849, "stderr": 7.456, "cost_per_test": 2.807198, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 71.418, "latency": 310.003, "stderr": 6.92, "cost_per_test": 0.517794, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 70.68, "latency": 3778.699, "stderr": 7.958, "cost_per_test": 10.250628, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 69.85, "latency": 551.297, "stderr": 7.875, "cost_per_test": 2.329763, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 68.713, "latency": 4910.699, "stderr": 7.789, "cost_per_test": 6.681535, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 68.577, "latency": 3072.683, "stderr": 7.764, "cost_per_test": 3.682319, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 68.547, "latency": 2805.95, "stderr": 7.745, "cost_per_test": 0.680751, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-4-argon": { "accuracy": 67.296, "latency": 1123.248, "stderr": 8.219, "cost_per_test": 4.046075, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 67.239, "latency": 1353.851, "stderr": 8.206, "cost_per_test": 7.081583, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 65.892, "latency": 2800.773, "stderr": 8.089, "cost_per_test": 4.479058, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 65.556, "latency": 906.405, "stderr": 7.479, "cost_per_test": 1.690921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 65.24, "latency": 1388.167, "stderr": 8.025, "cost_per_test": 2.360426, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 65.123, "latency": 207.207, "stderr": 8.013, "cost_per_test": 1.032835, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 64.903, "latency": 3219.385, "stderr": 7.976, "cost_per_test": 1.938554, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 64.183, "latency": 339.21, "stderr": 7.921, "cost_per_test": 0.669171, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 64.004, "latency": 1114.894, "stderr": 8.454, "cost_per_test": 9.612732, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 63.833, "latency": 4397.815, "stderr": 8.421, "cost_per_test": 17.005457, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 63.76, "latency": 1194.065, "stderr": 8.424, "cost_per_test": 2.217268, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 62.926, "latency": 2369.492, "stderr": 7.793, "cost_per_test": 0.262741, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 62.444, "latency": 1488.257, "stderr": 7.858, "cost_per_test": 0.881545, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 61.821, "latency": 3272.354, "stderr": 8.221, "cost_per_test": 1.431486, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 61.369, "latency": 1583.572, "stderr": 8.155, "cost_per_test": 0.163853, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 60.759, "latency": 1243.963, "stderr": 8.09, "cost_per_test": 8.333408, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 60.433, "latency": 388.616, "stderr": 8.038, "cost_per_test": 1.955097, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 59.946, "latency": 1233.388, "stderr": 8.012, "cost_per_test": 5.638196, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 59.514, "latency": 164.491, "stderr": 5.331, "cost_per_test": 0.387308, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 59.417, "latency": 400.472, "stderr": 7.946, "cost_per_test": 0.181934, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 59.147, "latency": 2048.707, "stderr": 8.434, "cost_per_test": 0.187946, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 58.822, "latency": 1628.156, "stderr": 7.488, "cost_per_test": 0.711742, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 58.465, "latency": 3355.896, "stderr": 8.315, "cost_per_test": 0.615879, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 58.15, "latency": 1588.793, "stderr": 8.339, "cost_per_test": 0.867778, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 57.72, "latency": 1823.192, "stderr": 8.264, "cost_per_test": 1.949409, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 56.777, "latency": 520.504, "stderr": 8.157, "cost_per_test": 0.215479, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 55.752, "latency": 638.305, "stderr": 8.509, "cost_per_test": 2.528215, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 54.86, "latency": 1141.66, "stderr": 7.965, "cost_per_test": 0.061032, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-sol": { "accuracy": 53.675, "latency": 1481.667, "stderr": 8.222, "cost_per_test": 3.18507, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 53.109, "latency": 356.103, "stderr": 7.776, "cost_per_test": 0.235599, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.369, "latency": 1812.809, "stderr": 8.093, "cost_per_test": 1.03018, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 50.385, "latency": 848.017, "stderr": 5.212, "cost_per_test": 0.290359, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 48.751, "latency": 125.77, "stderr": 8.555, "cost_per_test": 0.587181, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 46.225, "latency": 254.763, "stderr": 8.194, "cost_per_test": 0.589702, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 45.945, "latency": 292.436, "stderr": 8.634, "cost_per_test": 0.990018, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 42.966, "latency": 443.722, "stderr": 8.081, "cost_per_test": 0.032484, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 42.965, "latency": 424.78, "stderr": 6.828, "cost_per_test": 0.118047, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 42.449, "latency": 1231.408, "stderr": 7.31, "cost_per_test": 0.493899, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 42.065, "latency": 135.418, "stderr": 8.406, "cost_per_test": 0.425256, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 37.63, "latency": 1222.959, "stderr": 8.107, "cost_per_test": 0.78066, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 33.823, "latency": 136.347, "stderr": 7.342, "cost_per_test": 0.174303, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 33.491, "latency": 1635.017, "stderr": 7.717, "cost_per_test": 1.295762, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 31.655, "latency": 75.867, "stderr": 7.794, "cost_per_test": 0.112503, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 31.339, "latency": 547.636, "stderr": 6.366, "cost_per_test": 0.075732, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 31.055, "latency": 511.366, "stderr": 6.661, "cost_per_test": 0.441983, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 30.691, "latency": 139.219, "stderr": 7.268, "cost_per_test": 0.438674, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 30.139, "latency": 392.718, "stderr": 7.604, "cost_per_test": 0.134966, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 28.839, "latency": 112.756, "stderr": 6.34, "cost_per_test": 0.147642, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 27.521, "latency": 120.841, "stderr": 7.388, "cost_per_test": 0.1451, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 25.786, "latency": 226.784, "stderr": 6.028, "cost_per_test": 0.325413, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 22.122, "latency": 357.11, "stderr": 6.096, "cost_per_test": 0.105246, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 21.256, "latency": 675.993, "stderr": 6.083, "cost_per_test": 1.974378, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 19.137, "latency": 431.478, "stderr": 5.993, "cost_per_test": 0.127614, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 19.069, "latency": 639.986, "stderr": 6.11, "cost_per_test": 0.02572, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 14.077, "latency": 24.412, "stderr": 5.269, "cost_per_test": 0.017174, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 6.949, "latency": 41.899, "stderr": 3.443, "cost_per_test": 0.031988, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 6.661, "latency": 800.005, "stderr": 3.751, "cost_per_test": 0.028078, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__controversy_precedence_analysis": { "meta/muse-wonder-full": { "accuracy": 76.097, "latency": 1309.206, "stderr": 7.11, "cost_per_test": 2.63952, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 73.236, "latency": 1323.029, "stderr": 7.536, "cost_per_test": 0.472427, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder": { "accuracy": 71.362, "latency": 604.74, "stderr": 7.408, "cost_per_test": 2.510462, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 71.078, "latency": 2947.783, "stderr": 7.4, "cost_per_test": 1.886037, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 70.553, "latency": 2183.013, "stderr": 7.935, "cost_per_test": 17.029068, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 70.021, "latency": 1182.24, "stderr": 7.882, "cost_per_test": 5.663294, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 68.8, "latency": 3020.854, "stderr": 7.759, "cost_per_test": 3.745616, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 68.738, "latency": 330.347, "stderr": 7.869, "cost_per_test": 0.476801, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 68.503, "latency": 3046.844, "stderr": 7.769, "cost_per_test": 8.653012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 68.433, "latency": 971.384, "stderr": 7.727, "cost_per_test": 7.426143, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 68.102, "latency": 699.569, "stderr": 7.703, "cost_per_test": 1.075954, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 66.871, "latency": 4856.106, "stderr": 8.194, "cost_per_test": 18.106388, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 65.674, "latency": 1495.787, "stderr": 8.037, "cost_per_test": 0.147433, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 65.191, "latency": 4234.396, "stderr": 7.978, "cost_per_test": 11.036173, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 64.833, "latency": 4094.142, "stderr": 7.958, "cost_per_test": 5.443978, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 64.799, "latency": 1316.356, "stderr": 7.933, "cost_per_test": 1.303605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 63.603, "latency": 1204.529, "stderr": 8.39, "cost_per_test": 4.081424, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 62.693, "latency": 1111.152, "stderr": 8.291, "cost_per_test": 2.079234, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 62.591, "latency": 2610.393, "stderr": 8.263, "cost_per_test": 1.651978, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 61.219, "latency": 2724.08, "stderr": 8.138, "cost_per_test": 0.499628, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 60.859, "latency": 679.661, "stderr": 8.112, "cost_per_test": 0.368144, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 60.658, "latency": 296.318, "stderr": 8.039, "cost_per_test": 0.566258, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 59.234, "latency": 2124.134, "stderr": 8.438, "cost_per_test": 3.173128, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 59.143, "latency": 2580.136, "stderr": 8.401, "cost_per_test": 0.629331, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 59.001, "latency": 207.327, "stderr": 8.386, "cost_per_test": 0.923283, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 58.743, "latency": 562.149, "stderr": 8.347, "cost_per_test": 1.989157, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 58.616, "latency": 1027.021, "stderr": 8.342, "cost_per_test": 5.960847, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 57.949, "latency": 815.165, "stderr": 8.254, "cost_per_test": 1.403597, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 57.642, "latency": 338.034, "stderr": 8.209, "cost_per_test": 1.530941, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 57.41, "latency": 1200.277, "stderr": 8.215, "cost_per_test": 1.879269, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 55.733, "latency": 1050.002, "stderr": 8.088, "cost_per_test": 0.054776, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 54.91, "latency": 1096.628, "stderr": 8.417, "cost_per_test": 4.706236, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 54.843, "latency": 1303.848, "stderr": 7.987, "cost_per_test": 0.703688, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 54.423, "latency": 2538.969, "stderr": 8.33, "cost_per_test": 0.235158, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 54.091, "latency": 3496.18, "stderr": 8.306, "cost_per_test": 1.356768, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 52.802, "latency": 407.264, "stderr": 8.191, "cost_per_test": 0.145904, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 52.429, "latency": 366.09, "stderr": 8.069, "cost_per_test": 0.172383, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 52.416, "latency": 1181.176, "stderr": 8.56, "cost_per_test": 0.143783, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 51.856, "latency": 1354.622, "stderr": 8.047, "cost_per_test": 0.601855, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 51.549, "latency": 1814.296, "stderr": 8.018, "cost_per_test": 0.816262, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 50.662, "latency": 1129.044, "stderr": 8.313, "cost_per_test": 2.192608, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 50.042, "latency": 3907.926, "stderr": 8.713, "cost_per_test": 3.413393, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 48.286, "latency": 1372.497, "stderr": 8.451, "cost_per_test": 0.832712, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 46.686, "latency": 881.823, "stderr": 7.817, "cost_per_test": 0.376296, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 45.997, "latency": 109.553, "stderr": 8.598, "cost_per_test": 0.524274, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 45.848, "latency": 365.534, "stderr": 8.574, "cost_per_test": 0.169527, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 45.667, "latency": 826.163, "stderr": 6.659, "cost_per_test": 1.13869, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 44.39, "latency": 142.434, "stderr": 5.911, "cost_per_test": 0.301074, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 42.296, "latency": 1040.508, "stderr": 7.986, "cost_per_test": 0.568809, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 40.447, "latency": 255.015, "stderr": 8.104, "cost_per_test": 0.526926, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 37.362, "latency": 125.067, "stderr": 8.036, "cost_per_test": 0.358641, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 36.258, "latency": 342.732, "stderr": 7.568, "cost_per_test": 0.101653, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 34.598, "latency": 59.71, "stderr": 7.519, "cost_per_test": 0.092032, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 34.381, "latency": 140.944, "stderr": 7.382, "cost_per_test": 0.341242, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 34.209, "latency": 267.542, "stderr": 7.864, "cost_per_test": 0.960272, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 32.699, "latency": 424.4, "stderr": 7.217, "cost_per_test": 0.043785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 32.323, "latency": 366.514, "stderr": 7.944, "cost_per_test": 0.030442, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 32.174, "latency": 1056.728, "stderr": 7.445, "cost_per_test": 0.812549, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 31.723, "latency": 564.048, "stderr": 7.424, "cost_per_test": 0.02036, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 30.152, "latency": 408.237, "stderr": 7.662, "cost_per_test": 0.104081, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 30.14, "latency": 111.652, "stderr": 7.505, "cost_per_test": 0.100435, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 26.789, "latency": 277.282, "stderr": 6.388, "cost_per_test": 0.077802, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 24.946, "latency": 103.548, "stderr": 6.348, "cost_per_test": 0.122994, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 22.85, "latency": 117.479, "stderr": 6.685, "cost_per_test": 0.155222, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 22.664, "latency": 394.763, "stderr": 7.032, "cost_per_test": 0.316092, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 21.321, "latency": 295.62, "stderr": 5.866, "cost_per_test": 0.180353, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 20.775, "latency": 174.648, "stderr": 5.661, "cost_per_test": 0.261882, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 18.379, "latency": 352.585, "stderr": 6.359, "cost_per_test": 1.210646, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 11.494, "latency": 22.448, "stderr": 4.488, "cost_per_test": 0.014665, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 11.232, "latency": 2038.781, "stderr": 4.839, "cost_per_test": 0.045743, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 3.553, "latency": 37.426, "stderr": 2.47, "cost_per_test": 0.02667, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } } } }