{ "metadata": { "benchmark": "Finance Agent v2.1", "slug": "fabv2-1", "description": "Evaluating agents on core financial analyst tasks using the FAB v2 harness, on the reviewed FABv2.1 dataset", "benchmark_id": "finance_agent_v2_1", "family": "finance_agent", "version": "2.1", "updated": "2026-10-19", "dataset_type": "private", "industry": "finance", "tasks": { "overall": "Overall", "all_pass": "All-Pass", "general_qualitative_analysis": "General Qualitative Analysis", "general_quantitative_analysis": "General Quantitative Analysis", "market_analysis": "Market Analysis", "comparables": "Comparables", "precedents": "Precedents", "adjustments": "Adjustments", "earnings_analysis": "Earnings Analysis", "disclosure_analysis": "Disclosure Analysis", "financial_modeling": "Financial Modeling" }, "models": [ "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.7", "kimi/kimi-k3", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "minimax/MiniMax-M3", "minimax/MiniMax-M3.1-flash", "mistralai/mistral-large-4", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6.1-sol", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 27 }, "tasks": { "overall": { "anthropic/claude-opus-5-5": { "accuracy": 67.846, "latency": 1980.122, "stderr": 2.051, "cost_per_test": 8.908721, "token_totals": { "input_tokens": 711290297, "output_tokens": 106934250, "reasoning_tokens": 92568283, "cache_read_tokens": 362746415, "cache_write_tokens": 348503177 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 67.658, "latency": 793.03, "stderr": 2.0, "cost_per_test": 4.315759, "token_totals": { "input_tokens": 602790494, "output_tokens": 31627905, "reasoning_tokens": 14149611, "cache_read_tokens": 291037965, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 67.575, "latency": 850.298, "stderr": 1.98, "cost_per_test": 2.680843, "token_totals": { "input_tokens": 389317179, "output_tokens": 18193553, "reasoning_tokens": 11613004, "cache_read_tokens": 188800282, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 67.275, "latency": 2066.221, "stderr": 2.023, "cost_per_test": 6.950246, "token_totals": { "input_tokens": 1206037903, "output_tokens": 148942679, "reasoning_tokens": 130899975, "cache_read_tokens": 579490734, "cache_write_tokens": 626479999 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 65.761, "latency": 443.888, "stderr": 2.039, "cost_per_test": 1.991077, "token_totals": { "input_tokens": 1102307399, "output_tokens": 26345690, "reasoning_tokens": 18599088, "cache_read_tokens": 708933383, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 65.545, "latency": 310.129, "stderr": 2.031, "cost_per_test": 2.620705, "token_totals": { "input_tokens": 398379578, "output_tokens": 15303665, "reasoning_tokens": 9870222, "cache_read_tokens": 190235588, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 65.428, "latency": 960.248, "stderr": 2.024, "cost_per_test": 0.366574, "token_totals": { "input_tokens": 596577990, "output_tokens": 83546431, "reasoning_tokens": 74653360, "cache_read_tokens": 389922803, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 65.373, "latency": 1864.634, "stderr": 2.063, "cost_per_test": 1.637243, "token_totals": { "input_tokens": 1065081554, "output_tokens": 206334140, "reasoning_tokens": 194190119, "cache_read_tokens": 378369304, "cache_write_tokens": 686667014 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 65.004, "latency": 1205.572, "stderr": 2.01, "cost_per_test": 1.030939, "token_totals": { "input_tokens": 323459972, "output_tokens": 37030623, "reasoning_tokens": 30619069, "cache_read_tokens": 134111615, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 64.259, "latency": 682.01, "stderr": 2.015, "cost_per_test": 0.183321, "token_totals": { "input_tokens": 292706558, "output_tokens": 11189430, "reasoning_tokens": 6591988, "cache_read_tokens": 126914944, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 63.316, "latency": 936.347, "stderr": 2.084, "cost_per_test": 0.066534, "token_totals": { "input_tokens": 389007728, "output_tokens": 14596133, "reasoning_tokens": 8588760, "cache_read_tokens": 208996480, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 62.933, "latency": 909.212, "stderr": 2.028, "cost_per_test": 0.466261, "token_totals": { "input_tokens": 1025601275, "output_tokens": 94944123, "reasoning_tokens": 82978352, "cache_read_tokens": 721979776, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.817, "latency": 1145.078, "stderr": 2.112, "cost_per_test": 1.626543, "token_totals": { "input_tokens": 253535763, "output_tokens": 21994277, "reasoning_tokens": 16385214, "cache_read_tokens": 70737183, "cache_write_tokens": 182747233 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 61.623, "latency": 1502.748, "stderr": 2.059, "cost_per_test": 0.667449, "token_totals": { "input_tokens": 332001046, "output_tokens": 49959222, "reasoning_tokens": 42800144, "cache_read_tokens": 128979992, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 61.383, "latency": 986.069, "stderr": 2.023, "cost_per_test": 1.021615, "token_totals": { "input_tokens": 373286518, "output_tokens": 29255827, "reasoning_tokens": null, "cache_read_tokens": 140372060, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 60.665, "latency": 1612.347, "stderr": 2.056, "cost_per_test": 2.758047, "token_totals": { "input_tokens": 3043998798, "output_tokens": 14268191, "reasoning_tokens": 7831933, "cache_read_tokens": 2388431856, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 60.587, "latency": 296.095, "stderr": 2.032, "cost_per_test": 0.072473, "token_totals": { "input_tokens": 231343920, "output_tokens": 17576955, "reasoning_tokens": 13137649, "cache_read_tokens": 91246328, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 60.585, "latency": 652.292, "stderr": 2.114, "cost_per_test": 2.238652, "token_totals": { "input_tokens": 741416449, "output_tokens": 22618555, "reasoning_tokens": 15034963, "cache_read_tokens": 461563904, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 60.428, "latency": 1228.75, "stderr": 2.12, "cost_per_test": 7.823464, "token_totals": { "input_tokens": 248231715, "output_tokens": 22170755, "reasoning_tokens": 16637610, "cache_read_tokens": 77618056, "cache_write_tokens": 170564517 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 60.013, "latency": 875.039, "stderr": 2.09, "cost_per_test": 1.881233, "token_totals": { "input_tokens": 400595105, "output_tokens": 15119518, "reasoning_tokens": 9960408, "cache_read_tokens": 216957696, "cache_write_tokens": 182544978 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 58.9, "latency": 542.506, "stderr": 2.054, "cost_per_test": 0.575115, "token_totals": { "input_tokens": 407803894, "output_tokens": 20205838, "reasoning_tokens": 0, "cache_read_tokens": 214876672, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 58.826, "latency": 904.204, "stderr": 2.07, "cost_per_test": 0.158978, "token_totals": { "input_tokens": 469331836, "output_tokens": 54339785, "reasoning_tokens": 45403914, "cache_read_tokens": 234241552, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 57.116, "latency": 476.985, "stderr": 2.083, "cost_per_test": 0.82234, "token_totals": { "input_tokens": 380103725, "output_tokens": 27027936, "reasoning_tokens": 21919502, "cache_read_tokens": 187724800, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 56.841, "latency": 926.859, "stderr": 2.123, "cost_per_test": 0.120919, "token_totals": { "input_tokens": 393427873, "output_tokens": 45914348, "reasoning_tokens": 42042763, "cache_read_tokens": 189468255, "cache_write_tokens": 203907655 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 54.338, "latency": 189.319, "stderr": 2.024, "cost_per_test": 0.302923, "token_totals": { "input_tokens": 308376416, "output_tokens": 10174452, "reasoning_tokens": 3711622, "cache_read_tokens": 152983134, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 49.873, "latency": 573.747, "stderr": 2.116, "cost_per_test": 0.891309, "token_totals": { "input_tokens": 845699789, "output_tokens": 26758592, "reasoning_tokens": null, "cache_read_tokens": 667318528, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 44.776, "latency": 314.216, "stderr": 2.095, "cost_per_test": 0.207921, "token_totals": { "input_tokens": 641180329, "output_tokens": 23499273, "reasoning_tokens": null, "cache_read_tokens": 529986304, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "all_pass": { "google/gemini-4-argon": { "accuracy": 58.868, "latency": 793.03, "stderr": 2.168, "cost_per_test": 4.315759, "token_totals": { "input_tokens": 602790494, "output_tokens": 31627905, "reasoning_tokens": 14149611, "cache_read_tokens": 291037965, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 56.275, "latency": 1980.122, "stderr": 2.24, "cost_per_test": 8.908721, "token_totals": { "input_tokens": 711290297, "output_tokens": 106934250, "reasoning_tokens": 92568283, "cache_read_tokens": 362746415, "cache_write_tokens": 348503177 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 54.88, "latency": 850.298, "stderr": 2.2, "cost_per_test": 2.680843, "token_totals": { "input_tokens": 389317179, "output_tokens": 18193553, "reasoning_tokens": 11613004, "cache_read_tokens": 188800282, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 54.478, "latency": 443.888, "stderr": 2.218, "cost_per_test": 1.991077, "token_totals": { "input_tokens": 1102307399, "output_tokens": 26345690, "reasoning_tokens": 18599088, "cache_read_tokens": 708933383, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 54.447, "latency": 2066.221, "stderr": 2.213, "cost_per_test": 6.950246, "token_totals": { "input_tokens": 1206037903, "output_tokens": 148942679, "reasoning_tokens": 130899975, "cache_read_tokens": 579490734, "cache_write_tokens": 626479999 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 53.102, "latency": 310.129, "stderr": 2.212, "cost_per_test": 2.620705, "token_totals": { "input_tokens": 398379578, "output_tokens": 15303665, "reasoning_tokens": 9870222, "cache_read_tokens": 190235588, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 51.924, "latency": 1205.572, "stderr": 2.195, "cost_per_test": 1.030939, "token_totals": { "input_tokens": 323459972, "output_tokens": 37030623, "reasoning_tokens": 30619069, "cache_read_tokens": 134111615, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 51.772, "latency": 1864.634, "stderr": 2.201, "cost_per_test": 1.637243, "token_totals": { "input_tokens": 1065081554, "output_tokens": 206334140, "reasoning_tokens": 194190119, "cache_read_tokens": 378369304, "cache_write_tokens": 686667014 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 51.708, "latency": 960.248, "stderr": 2.209, "cost_per_test": 0.366574, "token_totals": { "input_tokens": 596577990, "output_tokens": 83546431, "reasoning_tokens": 74653360, "cache_read_tokens": 389922803, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 51.345, "latency": 909.212, "stderr": 2.198, "cost_per_test": 0.466261, "token_totals": { "input_tokens": 1025601275, "output_tokens": 94944123, "reasoning_tokens": 82978352, "cache_read_tokens": 721979776, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 50.672, "latency": 936.347, "stderr": 2.227, "cost_per_test": 0.066534, "token_totals": { "input_tokens": 389007728, "output_tokens": 14596133, "reasoning_tokens": 8588760, "cache_read_tokens": 208996480, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 50.437, "latency": 682.01, "stderr": 2.209, "cost_per_test": 0.183321, "token_totals": { "input_tokens": 292706558, "output_tokens": 11189430, "reasoning_tokens": 6591988, "cache_read_tokens": 126914944, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 49.242, "latency": 652.292, "stderr": 2.237, "cost_per_test": 2.238652, "token_totals": { "input_tokens": 741416449, "output_tokens": 22618555, "reasoning_tokens": 15034963, "cache_read_tokens": 461563904, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 49.05, "latency": 1502.748, "stderr": 2.214, "cost_per_test": 0.667449, "token_totals": { "input_tokens": 332001046, "output_tokens": 49959222, "reasoning_tokens": 42800144, "cache_read_tokens": 128979992, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-spark-open": { "accuracy": 48.957, "latency": 1612.347, "stderr": 2.204, "cost_per_test": 2.758047, "token_totals": { "input_tokens": 3043998798, "output_tokens": 14268191, "reasoning_tokens": 7831933, "cache_read_tokens": 2388431856, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 48.195, "latency": 1145.078, "stderr": 2.238, "cost_per_test": 1.626543, "token_totals": { "input_tokens": 253535763, "output_tokens": 21994277, "reasoning_tokens": 16385214, "cache_read_tokens": 70737183, "cache_write_tokens": 182747233 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 47.836, "latency": 296.095, "stderr": 2.177, "cost_per_test": 0.072473, "token_totals": { "input_tokens": 231343920, "output_tokens": 17576955, "reasoning_tokens": 13137649, "cache_read_tokens": 91246328, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 47.825, "latency": 986.069, "stderr": 2.189, "cost_per_test": 1.021615, "token_totals": { "input_tokens": 373286518, "output_tokens": 29255827, "reasoning_tokens": null, "cache_read_tokens": 140372060, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 47.461, "latency": 1228.75, "stderr": 2.223, "cost_per_test": 7.823464, "token_totals": { "input_tokens": 248231715, "output_tokens": 22170755, "reasoning_tokens": 16637610, "cache_read_tokens": 77618056, "cache_write_tokens": 170564517 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 47.306, "latency": 875.039, "stderr": 2.245, "cost_per_test": 1.881233, "token_totals": { "input_tokens": 400595105, "output_tokens": 15119518, "reasoning_tokens": 9960408, "cache_read_tokens": 216957696, "cache_write_tokens": 182544978 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 45.595, "latency": 904.204, "stderr": 2.206, "cost_per_test": 0.158978, "token_totals": { "input_tokens": 469331836, "output_tokens": 54339785, "reasoning_tokens": 45403914, "cache_read_tokens": 234241552, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "stepfun/step-5-preview": { "accuracy": 44.665, "latency": 542.506, "stderr": 2.162, "cost_per_test": 0.575115, "token_totals": { "input_tokens": 407803894, "output_tokens": 20205838, "reasoning_tokens": 0, "cache_read_tokens": 214876672, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 43.78, "latency": 926.859, "stderr": 2.214, "cost_per_test": 0.120919, "token_totals": { "input_tokens": 393427873, "output_tokens": 45914348, "reasoning_tokens": 42042763, "cache_read_tokens": 189468255, "cache_write_tokens": 203907655 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 42.497, "latency": 189.319, "stderr": 2.162, "cost_per_test": 0.302923, "token_totals": { "input_tokens": 308376416, "output_tokens": 10174452, "reasoning_tokens": 3711622, "cache_read_tokens": 152983134, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 42.117, "latency": 476.985, "stderr": 2.164, "cost_per_test": 0.82234, "token_totals": { "input_tokens": 380103725, "output_tokens": 27027936, "reasoning_tokens": 21919502, "cache_read_tokens": 187724800, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 36.308, "latency": 573.747, "stderr": 2.157, "cost_per_test": 0.891309, "token_totals": { "input_tokens": 845699789, "output_tokens": 26758592, "reasoning_tokens": null, "cache_read_tokens": 667318528, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 31.855, "latency": 314.216, "stderr": 2.093, "cost_per_test": 0.207921, "token_totals": { "input_tokens": 641180329, "output_tokens": 23499273, "reasoning_tokens": null, "cache_read_tokens": 529986304, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "general_qualitative_analysis": { "google/gemini-4-argon": { "accuracy": 79.151, "latency": 770.135, "stderr": 5.368, "cost_per_test": 4.221253, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 76.051, "latency": 1157.917, "stderr": 5.287, "cost_per_test": 3.202414, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 75.291, "latency": 1581.951, "stderr": 5.479, "cost_per_test": 2.006504, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 74.192, "latency": 703.694, "stderr": 5.223, "cost_per_test": 0.209884, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 73.855, "latency": 1265.243, "stderr": 5.425, "cost_per_test": 0.223672, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.3": { "accuracy": 73.552, "latency": 1340.681, "stderr": 5.436, "cost_per_test": 1.15972, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 73.185, "latency": 3579.94, "stderr": 5.674, "cost_per_test": 14.496451, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 72.992, "latency": 1087.084, "stderr": 5.431, "cost_per_test": 0.401506, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 71.674, "latency": 831.168, "stderr": 5.165, "cost_per_test": 1.035373, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 71.512, "latency": 3694.281, "stderr": 5.359, "cost_per_test": 11.759644, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 71.442, "latency": 418.321, "stderr": 5.622, "cost_per_test": 0.085413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 70.996, "latency": 377.222, "stderr": 5.843, "cost_per_test": 1.89482, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 70.995, "latency": 1143.475, "stderr": 5.818, "cost_per_test": 0.573931, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 70.096, "latency": 589.714, "stderr": 5.787, "cost_per_test": 2.023349, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 69.079, "latency": 472.07, "stderr": 5.671, "cost_per_test": 0.966847, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 69.061, "latency": 164.592, "stderr": 5.607, "cost_per_test": 0.315865, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 68.567, "latency": 422.263, "stderr": 5.503, "cost_per_test": 0.741564, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 68.483, "latency": 1075.012, "stderr": 5.679, "cost_per_test": 0.080161, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder": { "accuracy": 67.507, "latency": 403.948, "stderr": 6.097, "cost_per_test": 3.322322, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.959, "latency": 851.49, "stderr": 5.903, "cost_per_test": 2.014761, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 64.511, "latency": 1752.826, "stderr": 5.892, "cost_per_test": 2.213508, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 64.034, "latency": 706.512, "stderr": 5.919, "cost_per_test": 0.699713, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 63.553, "latency": 3406.468, "stderr": 6.089, "cost_per_test": 3.147282, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 62.876, "latency": 1983.771, "stderr": 6.03, "cost_per_test": 0.952051, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 61.906, "latency": 1895.772, "stderr": 5.968, "cost_per_test": 10.016985, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 57.736, "latency": 270.531, "stderr": 5.974, "cost_per_test": 0.163541, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 56.805, "latency": 1047.438, "stderr": 6.041, "cost_per_test": 0.142624, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "general_quantitative_analysis": { "anthropic/claude-haiku-5-5": { "accuracy": 87.806, "latency": 1390.142, "stderr": 4.021, "cost_per_test": 1.381808, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 84.886, "latency": 683.77, "stderr": 4.335, "cost_per_test": 2.504633, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 83.698, "latency": 649.808, "stderr": 4.772, "cost_per_test": 4.030901, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 83.468, "latency": 681.63, "stderr": 4.416, "cost_per_test": 0.211745, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder": { "accuracy": 82.207, "latency": 241.751, "stderr": 4.852, "cost_per_test": 2.384636, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 81.062, "latency": 1621.001, "stderr": 4.989, "cost_per_test": 6.029896, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 80.923, "latency": 1208.466, "stderr": 4.861, "cost_per_test": 0.070233, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 80.178, "latency": 401.728, "stderr": 4.977, "cost_per_test": 1.985641, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 79.509, "latency": 861.201, "stderr": 5.139, "cost_per_test": 6.73805, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 79.464, "latency": 998.507, "stderr": 5.025, "cost_per_test": 0.951176, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 79.378, "latency": 1491.808, "stderr": 5.051, "cost_per_test": 7.784115, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 78.899, "latency": 438.936, "stderr": 5.131, "cost_per_test": 0.867256, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 78.6, "latency": 306.528, "stderr": 5.035, "cost_per_test": 0.066929, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 78.577, "latency": 809.151, "stderr": 5.128, "cost_per_test": 1.300392, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 77.2, "latency": 630.645, "stderr": 5.2, "cost_per_test": 1.615485, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 76.517, "latency": 734.763, "stderr": 5.398, "cost_per_test": 0.333817, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 76.107, "latency": 1164.985, "stderr": 5.174, "cost_per_test": 1.027742, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 75.971, "latency": 720.164, "stderr": 5.258, "cost_per_test": 0.435356, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 75.883, "latency": 894.853, "stderr": 5.344, "cost_per_test": 1.160885, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 75.038, "latency": 1174.218, "stderr": 5.347, "cost_per_test": 0.57704, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "stepfun/step-5-preview": { "accuracy": 74.805, "latency": 378.592, "stderr": 5.34, "cost_per_test": 0.467778, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 74.288, "latency": 689.946, "stderr": 5.585, "cost_per_test": 0.099782, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 73.741, "latency": 722.092, "stderr": 5.482, "cost_per_test": 0.142236, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 70.506, "latency": 137.59, "stderr": 5.734, "cost_per_test": 0.260423, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.7": { "accuracy": 69.353, "latency": 486.685, "stderr": 5.946, "cost_per_test": 1.72208, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 64.567, "latency": 759.157, "stderr": 6.12, "cost_per_test": 1.025941, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 61.019, "latency": 299.98, "stderr": 5.945, "cost_per_test": 0.222562, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "market_analysis": { "anthropic/claude-sonnet-5-5": { "accuracy": 71.041, "latency": 1342.679, "stderr": 6.395, "cost_per_test": 4.047543, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 70.673, "latency": 550.697, "stderr": 6.617, "cost_per_test": 3.201461, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 70.484, "latency": 409.898, "stderr": 6.602, "cost_per_test": 1.357636, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 67.877, "latency": 227.501, "stderr": 6.706, "cost_per_test": 0.041813, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 67.795, "latency": 1354.839, "stderr": 6.707, "cost_per_test": 5.618355, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 67.685, "latency": 256.023, "stderr": 6.696, "cost_per_test": 1.433767, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 67.361, "latency": 520.944, "stderr": 6.694, "cost_per_test": 1.430893, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 67.081, "latency": 812.872, "stderr": 6.662, "cost_per_test": 0.258864, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 66.909, "latency": 941.895, "stderr": 6.65, "cost_per_test": 0.368951, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 66.589, "latency": 1215.818, "stderr": 6.62, "cost_per_test": 0.774829, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 65.172, "latency": 361.647, "stderr": 6.571, "cost_per_test": 0.302516, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-astra": { "accuracy": 64.22, "latency": 836.603, "stderr": 6.743, "cost_per_test": 5.051776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 64.03, "latency": 419.222, "stderr": 6.7, "cost_per_test": 1.418326, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 62.987, "latency": 305.051, "stderr": 6.834, "cost_per_test": 0.392298, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 62.2, "latency": 159.755, "stderr": 6.807, "cost_per_test": 0.205836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 61.389, "latency": 1070.997, "stderr": 6.753, "cost_per_test": 0.677645, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 60.979, "latency": 462.362, "stderr": 6.95, "cost_per_test": 0.093737, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 60.972, "latency": 886.591, "stderr": 6.97, "cost_per_test": 1.032567, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 60.972, "latency": 906.017, "stderr": 6.97, "cost_per_test": 0.709062, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 60.69, "latency": 637.261, "stderr": 6.951, "cost_per_test": 0.286289, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 56.642, "latency": 1287.19, "stderr": 7.026, "cost_per_test": 4.555468, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 55.738, "latency": 795.075, "stderr": 6.969, "cost_per_test": 0.125556, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 54.96, "latency": 739.781, "stderr": 7.126, "cost_per_test": 0.036127, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 52.38, "latency": 601.858, "stderr": 6.914, "cost_per_test": 0.084638, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 52.083, "latency": 641.225, "stderr": 7.287, "cost_per_test": 1.059659, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 49.847, "latency": 339.281, "stderr": 6.884, "cost_per_test": 0.658432, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 43.645, "latency": 234.416, "stderr": 6.669, "cost_per_test": 0.156945, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "comparables": { "google/gemini-4-argon": { "accuracy": 61.733, "latency": 963.322, "stderr": 6.462, "cost_per_test": 5.814253, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 59.751, "latency": 1064.803, "stderr": 6.375, "cost_per_test": 0.51929, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 59.248, "latency": 543.809, "stderr": 6.478, "cost_per_test": 2.583678, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 59.223, "latency": 1093.532, "stderr": 6.302, "cost_per_test": 2.682412, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 58.638, "latency": 906.644, "stderr": 6.474, "cost_per_test": 0.278101, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 58.452, "latency": 1542.203, "stderr": 6.494, "cost_per_test": 1.340498, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 57.899, "latency": 2365.612, "stderr": 6.641, "cost_per_test": 8.25005, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 57.714, "latency": 1965.792, "stderr": 6.454, "cost_per_test": 1.426828, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 57.276, "latency": 2035.43, "stderr": 6.648, "cost_per_test": 10.353098, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 57.226, "latency": 1604.519, "stderr": 6.403, "cost_per_test": 0.090569, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 57.036, "latency": 1878.316, "stderr": 6.566, "cost_per_test": 2.070621, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 56.745, "latency": 1457.409, "stderr": 6.389, "cost_per_test": 0.183196, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 55.818, "latency": 1133.071, "stderr": 6.481, "cost_per_test": 0.461555, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 55.011, "latency": 1094.974, "stderr": 6.639, "cost_per_test": 3.896655, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 54.967, "latency": 412.873, "stderr": 6.611, "cost_per_test": 3.823525, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 54.872, "latency": 917.131, "stderr": 6.714, "cost_per_test": 2.969299, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 54.458, "latency": 611.241, "stderr": 6.499, "cost_per_test": 1.155883, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 54.442, "latency": 2030.108, "stderr": 6.588, "cost_per_test": 1.599423, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 53.62, "latency": 683.515, "stderr": 6.479, "cost_per_test": 0.804444, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "tencent/hy4-preview": { "accuracy": 53.304, "latency": 1864.985, "stderr": 6.633, "cost_per_test": 0.924017, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 53.051, "latency": 359.824, "stderr": 6.553, "cost_per_test": 0.095835, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 53.008, "latency": 1155.838, "stderr": 6.676, "cost_per_test": 0.2003, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 50.115, "latency": 1747.651, "stderr": 6.51, "cost_per_test": 2.538897, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 50.084, "latency": 1981.022, "stderr": 6.508, "cost_per_test": 13.17291, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.479, "latency": 240.575, "stderr": 6.703, "cost_per_test": 0.401203, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 45.75, "latency": 811.434, "stderr": 6.519, "cost_per_test": 1.233939, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 42.485, "latency": 377.502, "stderr": 6.556, "cost_per_test": 0.260398, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "precedents": { "xiaomi/mimo-v2.6-flash": { "accuracy": 49.287, "latency": 1509.317, "stderr": 7.122, "cost_per_test": 0.078025, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 45.64, "latency": 2541.635, "stderr": 7.184, "cost_per_test": 10.956697, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 45.558, "latency": 451.591, "stderr": 7.199, "cost_per_test": 4.028488, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 44.181, "latency": 1253.085, "stderr": 7.015, "cost_per_test": 4.151451, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 43.482, "latency": 2563.522, "stderr": 7.149, "cost_per_test": 9.019741, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 43.109, "latency": 2180.825, "stderr": 7.095, "cost_per_test": 1.973177, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 42.742, "latency": 1579.501, "stderr": 7.074, "cost_per_test": 2.575052, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 40.388, "latency": 1405.413, "stderr": 6.989, "cost_per_test": 7.5572, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 40.185, "latency": 1695.727, "stderr": 6.96, "cost_per_test": 12.448809, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 39.312, "latency": 2003.732, "stderr": 7.046, "cost_per_test": 0.899681, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-spark-open": { "accuracy": 39.255, "latency": 2050.598, "stderr": 7.053, "cost_per_test": 3.945511, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 39.129, "latency": 901.071, "stderr": 7.035, "cost_per_test": 0.24417, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 38.923, "latency": 699.24, "stderr": 7.011, "cost_per_test": 3.058619, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 38.868, "latency": 2089.096, "stderr": 6.987, "cost_per_test": 1.784257, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 38.562, "latency": 382.597, "stderr": 6.961, "cost_per_test": 0.097667, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 37.933, "latency": 1408.585, "stderr": 6.848, "cost_per_test": 0.547168, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 36.957, "latency": 1531.111, "stderr": 6.984, "cost_per_test": 0.19807, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 36.722, "latency": 1465.787, "stderr": 6.906, "cost_per_test": 1.301607, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 35.402, "latency": 1154.136, "stderr": 6.955, "cost_per_test": 5.088313, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 33.919, "latency": 1225.044, "stderr": 6.7, "cost_per_test": 0.222412, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "kimi/kimi-k3": { "accuracy": 32.576, "latency": 1446.226, "stderr": 6.734, "cost_per_test": 3.411789, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 31.079, "latency": 782.372, "stderr": 6.701, "cost_per_test": 0.905855, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 30.713, "latency": 665.958, "stderr": 6.638, "cost_per_test": 1.098715, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 30.19, "latency": 836.474, "stderr": 6.565, "cost_per_test": 1.178546, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 29.348, "latency": 1361.053, "stderr": 6.67, "cost_per_test": 0.709077, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 18.601, "latency": 308.135, "stderr": 5.682, "cost_per_test": 0.502391, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 14.159, "latency": 445.513, "stderr": 5.046, "cost_per_test": 0.272102, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "adjustments": { "meta/muse-wonder-full": { "accuracy": 74.737, "latency": 888.018, "stderr": 6.048, "cost_per_test": 2.686259, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 70.813, "latency": 730.593, "stderr": 6.365, "cost_per_test": 4.008119, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 67.652, "latency": 1836.366, "stderr": 6.634, "cost_per_test": 1.465053, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 67.476, "latency": 1239.889, "stderr": 6.619, "cost_per_test": 1.038099, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 67.068, "latency": 1855.643, "stderr": 6.597, "cost_per_test": 9.062921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 66.771, "latency": 954.029, "stderr": 6.591, "cost_per_test": 0.506283, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 66.72, "latency": 1594.395, "stderr": 6.562, "cost_per_test": 0.643717, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 66.589, "latency": 947.806, "stderr": 6.575, "cost_per_test": 0.361027, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 66.229, "latency": 1959.63, "stderr": 6.561, "cost_per_test": 6.612696, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 64.735, "latency": 292.846, "stderr": 6.669, "cost_per_test": 2.505207, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 62.995, "latency": 1335.692, "stderr": 6.801, "cost_per_test": 2.51918, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 61.905, "latency": 460.111, "stderr": 6.726, "cost_per_test": 1.87931, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 61.78, "latency": 981.59, "stderr": 6.72, "cost_per_test": 2.058485, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 59.658, "latency": 812.371, "stderr": 6.963, "cost_per_test": 0.144584, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 59.644, "latency": 1053.468, "stderr": 6.961, "cost_per_test": 0.781985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 59.501, "latency": 161.247, "stderr": 6.946, "cost_per_test": 0.272319, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 59.112, "latency": 542.344, "stderr": 6.932, "cost_per_test": 0.147747, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 58.544, "latency": 603.135, "stderr": 6.886, "cost_per_test": 1.995259, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 58.495, "latency": 563.336, "stderr": 6.891, "cost_per_test": 0.586728, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-astra": { "accuracy": 58.136, "latency": 1051.395, "stderr": 6.831, "cost_per_test": 6.489114, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 57.636, "latency": 280.927, "stderr": 7.011, "cost_per_test": 0.072972, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 57.337, "latency": 837.411, "stderr": 6.982, "cost_per_test": 0.071633, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 56.278, "latency": 808.317, "stderr": 6.889, "cost_per_test": 0.115355, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 54.722, "latency": 1019.859, "stderr": 6.97, "cost_per_test": 1.548504, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 51.394, "latency": 464.421, "stderr": 7.062, "cost_per_test": 0.827677, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 42.049, "latency": 559.98, "stderr": 6.831, "cost_per_test": 0.789789, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 39.373, "latency": 402.845, "stderr": 6.899, "cost_per_test": 0.230261, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "earnings_analysis": { "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 93.029, "latency": 565.155, "stderr": 3.411, "cost_per_test": 0.22062, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 92.409, "latency": 708.946, "stderr": 3.476, "cost_per_test": 0.436133, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder": { "accuracy": 90.331, "latency": 161.195, "stderr": 3.884, "cost_per_test": 0.855159, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 88.886, "latency": 1444.703, "stderr": 4.289, "cost_per_test": 4.822133, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 88.503, "latency": 345.617, "stderr": 4.364, "cost_per_test": 0.057919, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 88.409, "latency": 338.892, "stderr": 4.353, "cost_per_test": 1.022523, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 88.256, "latency": 650.286, "stderr": 4.276, "cost_per_test": 0.198487, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 86.503, "latency": 1103.738, "stderr": 4.702, "cost_per_test": 2.739073, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 85.131, "latency": 376.22, "stderr": 4.722, "cost_per_test": 0.219532, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 85.029, "latency": 298.909, "stderr": 4.948, "cost_per_test": 0.024324, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder-full": { "accuracy": 84.794, "latency": 436.722, "stderr": 4.939, "cost_per_test": 1.018528, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 84.503, "latency": 355.923, "stderr": 5.0, "cost_per_test": 0.833315, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 84.503, "latency": 1112.602, "stderr": 5.0, "cost_per_test": 0.318611, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 84.259, "latency": 138.738, "stderr": 4.936, "cost_per_test": 0.025676, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 84.044, "latency": 432.126, "stderr": 4.983, "cost_per_test": 0.049697, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 83.851, "latency": 466.585, "stderr": 4.946, "cost_per_test": 0.481907, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 82.632, "latency": 660.143, "stderr": 5.266, "cost_per_test": 2.383254, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 82.503, "latency": 1113.458, "stderr": 5.267, "cost_per_test": 0.299345, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 82.124, "latency": 643.947, "stderr": 5.244, "cost_per_test": 0.078555, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "meta/muse-spark-open": { "accuracy": 81.263, "latency": 1037.852, "stderr": 5.468, "cost_per_test": 1.221736, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 80.503, "latency": 908.513, "stderr": 5.505, "cost_per_test": 0.433533, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 80.404, "latency": 504.445, "stderr": 5.425, "cost_per_test": 0.674956, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 80.086, "latency": 523.24, "stderr": 5.459, "cost_per_test": 2.388251, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 74.371, "latency": 120.285, "stderr": 6.07, "cost_per_test": 0.11551, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 74.253, "latency": 372.13, "stderr": 6.063, "cost_per_test": 0.365709, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 64.947, "latency": 225.405, "stderr": 6.706, "cost_per_test": 0.117843, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 63.846, "latency": 304.675, "stderr": 6.842, "cost_per_test": 0.400981, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "disclosure_analysis": { "google/gemini-4-argon": { "accuracy": 83.345, "latency": 724.888, "stderr": 4.983, "cost_per_test": 4.320175, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 80.595, "latency": 962.739, "stderr": 5.226, "cost_per_test": 3.162266, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 78.79, "latency": 2963.742, "stderr": 5.459, "cost_per_test": 10.120528, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 78.189, "latency": 349.46, "stderr": 5.455, "cost_per_test": 3.355491, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "tencent/hy4-preview": { "accuracy": 77.6, "latency": 1910.587, "stderr": 5.413, "cost_per_test": 0.876339, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 77.016, "latency": 1427.752, "stderr": 5.685, "cost_per_test": 1.313999, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 76.903, "latency": 1147.1, "stderr": 5.681, "cost_per_test": 0.459868, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 76.791, "latency": 1369.397, "stderr": 5.676, "cost_per_test": 1.846256, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 76.582, "latency": 414.514, "stderr": 5.689, "cost_per_test": 2.020335, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 75.432, "latency": 476.067, "stderr": 5.898, "cost_per_test": 0.08808, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 75.307, "latency": 747.243, "stderr": 5.596, "cost_per_test": 0.236172, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 74.406, "latency": 2447.919, "stderr": 5.848, "cost_per_test": 11.310446, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 73.501, "latency": 1312.711, "stderr": 5.825, "cost_per_test": 1.169946, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 71.787, "latency": 2872.991, "stderr": 5.982, "cost_per_test": 2.697765, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 71.634, "latency": 1120.854, "stderr": 6.006, "cost_per_test": 2.362115, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 71.221, "latency": 628.602, "stderr": 6.218, "cost_per_test": 2.09646, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 70.536, "latency": 318.275, "stderr": 5.922, "cost_per_test": 0.102454, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 70.509, "latency": 1102.958, "stderr": 6.171, "cost_per_test": 0.19942, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 70.262, "latency": 1178.026, "stderr": 6.153, "cost_per_test": 0.654078, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 70.097, "latency": 200.972, "stderr": 5.966, "cost_per_test": 0.310624, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "stepfun/step-5-preview": { "accuracy": 68.434, "latency": 587.155, "stderr": 6.344, "cost_per_test": 0.706785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 68.257, "latency": 1222.515, "stderr": 6.321, "cost_per_test": 1.921714, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 65.085, "latency": 531.616, "stderr": 6.335, "cost_per_test": 0.997146, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 64.698, "latency": 1360.542, "stderr": 6.568, "cost_per_test": 9.009417, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 64.314, "latency": 475.645, "stderr": 6.311, "cost_per_test": 0.766621, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 57.587, "latency": 1030.105, "stderr": 6.744, "cost_per_test": 0.128071, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 54.622, "latency": 307.902, "stderr": 6.662, "cost_per_test": 0.211482, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "financial_modeling": { "anthropic/claude-opus-5-5": { "accuracy": 56.983, "latency": 1284.64, "stderr": 6.794, "cost_per_test": 6.466784, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 52.602, "latency": 950.532, "stderr": 6.803, "cost_per_test": 1.199828, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 51.518, "latency": 1164.571, "stderr": 6.902, "cost_per_test": 1.090725, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 48.954, "latency": 1172.341, "stderr": 6.834, "cost_per_test": 4.457192, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 46.767, "latency": 836.605, "stderr": 6.801, "cost_per_test": 0.301129, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 45.121, "latency": 522.861, "stderr": 6.823, "cost_per_test": 2.209835, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 45.024, "latency": 998.948, "stderr": 6.798, "cost_per_test": 5.911616, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 42.331, "latency": 1115.43, "stderr": 6.661, "cost_per_test": 0.986011, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 41.169, "latency": 735.773, "stderr": 6.727, "cost_per_test": 0.060752, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder-full": { "accuracy": 40.557, "latency": 735.548, "stderr": 6.661, "cost_per_test": 2.261304, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 39.583, "latency": 668.471, "stderr": 6.73, "cost_per_test": 0.318493, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 39.003, "latency": 921.222, "stderr": 6.645, "cost_per_test": 0.174077, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder": { "accuracy": 38.73, "latency": 264.44, "stderr": 6.612, "cost_per_test": 2.078374, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 38.259, "latency": 712.206, "stderr": 6.575, "cost_per_test": 1.237662, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 37.243, "latency": 809.69, "stderr": 6.611, "cost_per_test": 2.318672, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 36.493, "latency": 776.73, "stderr": 6.717, "cost_per_test": 3.668479, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 36.481, "latency": 865.535, "stderr": 6.505, "cost_per_test": 0.099256, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 30.345, "latency": 1059.78, "stderr": 6.375, "cost_per_test": 0.509733, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "stepfun/step-5-preview": { "accuracy": 29.33, "latency": 508.535, "stderr": 6.184, "cost_per_test": 0.53266, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 27.173, "latency": 468.733, "stderr": 5.759, "cost_per_test": 0.752164, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 26.881, "latency": 513.254, "stderr": 5.969, "cost_per_test": 0.104852, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 24.996, "latency": 313.469, "stderr": 5.883, "cost_per_test": 0.239496, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 24.494, "latency": 1112.518, "stderr": 5.719, "cost_per_test": 0.928166, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 23.42, "latency": 3287.51, "stderr": 5.749, "cost_per_test": 6.232657, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 23.318, "latency": 265.45, "stderr": 5.69, "cost_per_test": 0.067281, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 19.725, "latency": 687.335, "stderr": 5.354, "cost_per_test": 1.229394, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 16.228, "latency": 270.009, "stderr": 4.814, "cost_per_test": 0.365806, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null } } } }