{ "metadata": { "benchmark": "Vals Finance Index", "slug": "finance_index", "description": "Agentic model performance on finance work, from analyst research and financial modeling to corporate tax research, weighted by each sub-sector's share of U.S. GDP.", "benchmark_id": "finance_index", "family": "finance_index", "version": "1", "updated": "2026-10-21", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v2.1", "emb": "EMB v1.1", "tax_agent_bench": "Tax Agent Bench" }, "models": [ "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.7", "kimi/kimi-k3", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "minimax/MiniMax-M3", "minimax/MiniMax-M3.1-flash", "mistralai/mistral-large-4", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6.1-sol", "reflection/Beam-1.0", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 28 }, "tasks": { "overall": { "anthropic/claude-opus-5-5": { "accuracy": 65.587, "latency": 2699.701, "stderr": 1.478, "cost_per_test": 11.671293, "token_totals": { "input_tokens": 2448837440, "output_tokens": 218639291, "reasoning_tokens": 187840339, "cache_read_tokens": 1872892770, "cache_write_tokens": 575851171 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 64.89, "latency": 1361.495, "stderr": 1.487, "cost_per_test": 6.530768, "token_totals": { "input_tokens": 2015769916, "output_tokens": 60379347, "reasoning_tokens": 29793337, "cache_read_tokens": 1436468850, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 64.551, "latency": 1206.366, "stderr": 1.399, "cost_per_test": 3.136769, "token_totals": { "input_tokens": 1630176345, "output_tokens": 36492232, "reasoning_tokens": 22799297, "cache_read_tokens": 1359223014, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 64.533, "latency": 2610.585, "stderr": 1.482, "cost_per_test": 8.329525, "token_totals": { "input_tokens": 4724854462, "output_tokens": 286300901, "reasoning_tokens": 247419997, "cache_read_tokens": 3845065220, "cache_write_tokens": 879653248 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 61.225, "latency": 612.47, "stderr": 1.458, "cost_per_test": 3.36816, "token_totals": { "input_tokens": 1696873775, "output_tokens": 32203153, "reasoning_tokens": 20382686, "cache_read_tokens": 1400008612, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 60.886, "latency": 1381.778, "stderr": 1.467, "cost_per_test": 0.418879, "token_totals": { "input_tokens": 3613967269, "output_tokens": 164623043, "reasoning_tokens": 140204906, "cache_read_tokens": 3231149301, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 60.544, "latency": 2143.578, "stderr": 1.519, "cost_per_test": 2.206579, "token_totals": { "input_tokens": 1494939959, "output_tokens": 102625607, "reasoning_tokens": null, "cache_read_tokens": 1179942762, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 60.461, "latency": 463.805, "stderr": 1.449, "cost_per_test": 3.697384, "token_totals": { "input_tokens": 4325623093, "output_tokens": 55081206, "reasoning_tokens": 36551704, "cache_read_tokens": 3704617525, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 58.459, "latency": 1248.595, "stderr": 1.449, "cost_per_test": 3.505492, "token_totals": { "input_tokens": 1553788141, "output_tokens": 48969812, "reasoning_tokens": 32293466, "cache_read_tokens": 1096486400, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 57.955, "latency": 1343.873, "stderr": 1.428, "cost_per_test": 2.582005, "token_totals": { "input_tokens": 1254906285, "output_tokens": 39813499, "reasoning_tokens": 27723237, "cache_read_tokens": 991918608, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 57.847, "latency": 1847.876, "stderr": 1.566, "cost_per_test": 0.215542, "token_totals": { "input_tokens": 1276605558, "output_tokens": 41266178, "reasoning_tokens": 29470584, "cache_read_tokens": 1043330944, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 57.507, "latency": 1935.395, "stderr": 1.449, "cost_per_test": 0.680041, "token_totals": { "input_tokens": 1338014986, "output_tokens": 89294342, "reasoning_tokens": 74253989, "cache_read_tokens": 1068525272, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 57.448, "latency": 2011.269, "stderr": 1.798, "cost_per_test": 1.38867, "token_totals": { "input_tokens": 3330005929, "output_tokens": 374700516, "reasoning_tokens": 347238361, "cache_read_tokens": 2437177857, "cache_write_tokens": 892717113 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 57.149, "latency": 1165.727, "stderr": 1.391, "cost_per_test": 0.074717, "token_totals": { "input_tokens": 1448678240, "output_tokens": 42784485, "reasoning_tokens": 28596950, "cache_read_tokens": 1198511104, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 57.078, "latency": 696.594, "stderr": 1.452, "cost_per_test": 0.245129, "token_totals": { "input_tokens": 1604495560, "output_tokens": 112392391, "reasoning_tokens": 97567561, "cache_read_tokens": 1366511984, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 56.895, "latency": 3109.826, "stderr": 1.417, "cost_per_test": 2.860726, "token_totals": { "input_tokens": 1701445396, "output_tokens": 73582654, "reasoning_tokens": null, "cache_read_tokens": 937726044, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 55.906, "latency": 1571.585, "stderr": 1.42, "cost_per_test": 0.249628, "token_totals": { "input_tokens": 2148215839, "output_tokens": 138838586, "reasoning_tokens": 118164640, "cache_read_tokens": 1813168693, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 55.832, "latency": 2029.434, "stderr": 1.511, "cost_per_test": 0.307856, "token_totals": { "input_tokens": 4358208100, "output_tokens": 75404498, "reasoning_tokens": null, "cache_read_tokens": 4080336888, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 55.81, "latency": 1155.589, "stderr": 1.499, "cost_per_test": 1.364448, "token_totals": { "input_tokens": 532475272, "output_tokens": 40768635, "reasoning_tokens": 29368899, "cache_read_tokens": 287098635, "cache_write_tokens": 245256298 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 55.642, "latency": 977.872, "stderr": 1.379, "cost_per_test": 6.482276, "token_totals": { "input_tokens": 488034216, "output_tokens": 39746120, "reasoning_tokens": 28854223, "cache_read_tokens": 253595918, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 55.428, "latency": 1482.039, "stderr": 1.435, "cost_per_test": 0.819271, "token_totals": { "input_tokens": 1981615249, "output_tokens": 51310340, "reasoning_tokens": 0, "cache_read_tokens": 1704772096, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 53.052, "latency": 1321.216, "stderr": 1.434, "cost_per_test": 0.151194, "token_totals": { "input_tokens": 1204816834, "output_tokens": 97585863, "reasoning_tokens": 87832544, "cache_read_tokens": 890199236, "cache_write_tokens": 314424282 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 52.99, "latency": 2464.262, "stderr": 1.44, "cost_per_test": 2.996809, "token_totals": { "input_tokens": 4430732068, "output_tokens": 26335421, "reasoning_tokens": 14388276, "cache_read_tokens": 3360685728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.94, "latency": 1440.325, "stderr": 1.405, "cost_per_test": 0.877629, "token_totals": { "input_tokens": 1417273009, "output_tokens": 63341911, "reasoning_tokens": 50997941, "cache_read_tokens": 1176627456, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.366, "latency": 600.9, "stderr": 1.413, "cost_per_test": 1.216942, "token_totals": { "input_tokens": 2561462789, "output_tokens": 29439464, "reasoning_tokens": 8373830, "cache_read_tokens": 2321904313, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "reflection/Beam-1.0": { "accuracy": 46.501, "latency": 449.231, "stderr": 1.261, "cost_per_test": null, "token_totals": { "input_tokens": 1519203196, "output_tokens": 25571090, "reasoning_tokens": 17651430, "cache_read_tokens": 1228536384, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 38.248, "latency": 988.713, "stderr": 1.508, "cost_per_test": 0.972283, "token_totals": { "input_tokens": 1651595449, "output_tokens": 43310194, "reasoning_tokens": null, "cache_read_tokens": 1439390720, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 36.273, "latency": 624.364, "stderr": 1.565, "cost_per_test": 0.182996, "token_totals": { "input_tokens": 1003859675, "output_tokens": 37291823, "reasoning_tokens": null, "cache_read_tokens": 873588480, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "finance_agent": { "anthropic/claude-opus-5-5": { "accuracy": 67.846, "latency": 1980.122, "stderr": 2.051, "cost_per_test": 8.908721, "token_totals": { "input_tokens": 711290297, "output_tokens": 106934250, "reasoning_tokens": 92568283, "cache_read_tokens": 362746415, "cache_write_tokens": 348503177 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 67.658, "latency": 793.03, "stderr": 2.0, "cost_per_test": 4.315759, "token_totals": { "input_tokens": 602790494, "output_tokens": 31627905, "reasoning_tokens": 14149611, "cache_read_tokens": 291037965, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 67.575, "latency": 850.298, "stderr": 1.98, "cost_per_test": 2.680843, "token_totals": { "input_tokens": 389317179, "output_tokens": 18193553, "reasoning_tokens": 11613004, "cache_read_tokens": 188800282, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 67.275, "latency": 2066.221, "stderr": 2.023, "cost_per_test": 6.950246, "token_totals": { "input_tokens": 1206037903, "output_tokens": 148942679, "reasoning_tokens": 130899975, "cache_read_tokens": 579490734, "cache_write_tokens": 626479999 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 65.761, "latency": 443.888, "stderr": 2.039, "cost_per_test": 1.991077, "token_totals": { "input_tokens": 1102307399, "output_tokens": 26345690, "reasoning_tokens": 18599088, "cache_read_tokens": 708933383, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 65.545, "latency": 310.129, "stderr": 2.031, "cost_per_test": 2.620705, "token_totals": { "input_tokens": 398379578, "output_tokens": 15303665, "reasoning_tokens": 9870222, "cache_read_tokens": 190235588, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 65.428, "latency": 960.248, "stderr": 2.024, "cost_per_test": 0.366574, "token_totals": { "input_tokens": 596577990, "output_tokens": 83546431, "reasoning_tokens": 74653360, "cache_read_tokens": 389922803, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 65.373, "latency": 1864.634, "stderr": 2.063, "cost_per_test": 1.637243, "token_totals": { "input_tokens": 1065081554, "output_tokens": 206334140, "reasoning_tokens": 194190119, "cache_read_tokens": 378369304, "cache_write_tokens": 686667014 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 65.004, "latency": 1205.572, "stderr": 2.01, "cost_per_test": 1.030939, "token_totals": { "input_tokens": 323459972, "output_tokens": 37030623, "reasoning_tokens": 30619069, "cache_read_tokens": 134111615, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 64.259, "latency": 682.01, "stderr": 2.015, "cost_per_test": 0.183321, "token_totals": { "input_tokens": 292706558, "output_tokens": 11189430, "reasoning_tokens": 6591988, "cache_read_tokens": 126914944, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 63.316, "latency": 936.347, "stderr": 2.084, "cost_per_test": 0.066534, "token_totals": { "input_tokens": 389007728, "output_tokens": 14596133, "reasoning_tokens": 8588760, "cache_read_tokens": 208996480, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 62.933, "latency": 909.212, "stderr": 2.028, "cost_per_test": 0.466261, "token_totals": { "input_tokens": 1025601275, "output_tokens": 94944123, "reasoning_tokens": 82978352, "cache_read_tokens": 721979776, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.817, "latency": 1145.078, "stderr": 2.112, "cost_per_test": 1.626543, "token_totals": { "input_tokens": 253535763, "output_tokens": 21994277, "reasoning_tokens": 16385214, "cache_read_tokens": 70737183, "cache_write_tokens": 182747233 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 61.623, "latency": 1502.748, "stderr": 2.059, "cost_per_test": 0.667449, "token_totals": { "input_tokens": 332001046, "output_tokens": 49959222, "reasoning_tokens": 42800144, "cache_read_tokens": 128979992, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 61.383, "latency": 986.069, "stderr": 2.023, "cost_per_test": 1.021615, "token_totals": { "input_tokens": 373286518, "output_tokens": 29255827, "reasoning_tokens": null, "cache_read_tokens": 140372060, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 60.665, "latency": 1612.347, "stderr": 2.056, "cost_per_test": 2.758047, "token_totals": { "input_tokens": 3043998798, "output_tokens": 14268191, "reasoning_tokens": 7831933, "cache_read_tokens": 2388431856, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 60.587, "latency": 296.095, "stderr": 2.032, "cost_per_test": 0.072473, "token_totals": { "input_tokens": 231343920, "output_tokens": 17576955, "reasoning_tokens": 13137649, "cache_read_tokens": 91246328, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 60.585, "latency": 652.292, "stderr": 2.114, "cost_per_test": 2.238652, "token_totals": { "input_tokens": 741416449, "output_tokens": 22618555, "reasoning_tokens": 15034963, "cache_read_tokens": 461563904, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 60.428, "latency": 1228.75, "stderr": 2.12, "cost_per_test": 7.823464, "token_totals": { "input_tokens": 248231715, "output_tokens": 22170755, "reasoning_tokens": 16637610, "cache_read_tokens": 77618056, "cache_write_tokens": 170564517 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 60.013, "latency": 875.039, "stderr": 2.09, "cost_per_test": 1.881233, "token_totals": { "input_tokens": 400595105, "output_tokens": 15119518, "reasoning_tokens": 9960408, "cache_read_tokens": 216957696, "cache_write_tokens": 182544978 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 58.9, "latency": 542.506, "stderr": 2.054, "cost_per_test": 0.575115, "token_totals": { "input_tokens": 407803894, "output_tokens": 20205838, "reasoning_tokens": 0, "cache_read_tokens": 214876672, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 58.826, "latency": 904.204, "stderr": 2.07, "cost_per_test": 0.158978, "token_totals": { "input_tokens": 469331836, "output_tokens": 54339785, "reasoning_tokens": 45403914, "cache_read_tokens": 234241552, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 57.116, "latency": 476.985, "stderr": 2.083, "cost_per_test": 0.82234, "token_totals": { "input_tokens": 380103725, "output_tokens": 27027936, "reasoning_tokens": 21919502, "cache_read_tokens": 187724800, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 56.841, "latency": 926.859, "stderr": 2.123, "cost_per_test": 0.120919, "token_totals": { "input_tokens": 393427873, "output_tokens": 45914348, "reasoning_tokens": 42042763, "cache_read_tokens": 189468255, "cache_write_tokens": 203907655 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 54.338, "latency": 189.319, "stderr": 2.024, "cost_per_test": 0.302923, "token_totals": { "input_tokens": 308376416, "output_tokens": 10174452, "reasoning_tokens": 3711622, "cache_read_tokens": 152983134, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "reflection/Beam-1.0": { "accuracy": 51.827, "latency": 85.425, "stderr": 2.107, "cost_per_test": null, "token_totals": { "input_tokens": 503284975, "output_tokens": 15066321, "reasoning_tokens": 11184585, "cache_read_tokens": 255618816, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 49.873, "latency": 573.747, "stderr": 2.116, "cost_per_test": 0.891309, "token_totals": { "input_tokens": 845699789, "output_tokens": 26758592, "reasoning_tokens": null, "cache_read_tokens": 667318528, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 44.776, "latency": 314.216, "stderr": 2.095, "cost_per_test": 0.207921, "token_totals": { "input_tokens": 641180329, "output_tokens": 23499273, "reasoning_tokens": null, "cache_read_tokens": 529986304, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "emb": { "anthropic/claude-sonnet-5-5": { "accuracy": 75.899, "latency": 2204.991, "stderr": 2.468, "cost_per_test": 8.792863, "token_totals": { "input_tokens": 2367951142, "output_tokens": 29328538, "reasoning_tokens": 22320203, "cache_read_tokens": 2311427347, "cache_write_tokens": 56507215 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.836, "latency": 2106.434, "stderr": 2.41, "cost_per_test": 11.018974, "token_totals": { "input_tokens": 922243024, "output_tokens": 23704546, "reasoning_tokens": 18569025, "cache_read_tokens": 861165257, "cache_write_tokens": 61067293 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 74.998, "latency": 1684.504, "stderr": 2.141, "cost_per_test": 4.58223, "token_totals": { "input_tokens": 983252171, "output_tokens": 8330149, "reasoning_tokens": 5099887, "cache_read_tokens": 955848482, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 74.828, "latency": 2305.821, "stderr": 2.494, "cost_per_test": 11.8029, "token_totals": { "input_tokens": 1127460088, "output_tokens": 14257556, "reasoning_tokens": 9497334, "cache_read_tokens": 945025185, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 72.871, "latency": 769.197, "stderr": 2.424, "cost_per_test": 8.243862, "token_totals": { "input_tokens": 2960728904, "output_tokens": 21722815, "reasoning_tokens": 13162823, "cache_read_tokens": 2787511478, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 72.683, "latency": 804.28, "stderr": 2.297, "cost_per_test": 5.827063, "token_totals": { "input_tokens": 123989946, "output_tokens": 5542631, "reasoning_tokens": 3051513, "cache_read_tokens": 104449111, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 72.503, "latency": 1034.437, "stderr": 2.414, "cost_per_test": 5.599337, "token_totals": { "input_tokens": 1072958990, "output_tokens": 8128857, "reasoning_tokens": 5046443, "cache_read_tokens": 1021915980, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 71.498, "latency": 2097.023, "stderr": 2.328, "cost_per_test": 6.481284, "token_totals": { "input_tokens": 531961928, "output_tokens": 15529729, "reasoning_tokens": 10549516, "cache_read_tokens": 448911360, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 71.264, "latency": 741.42, "stderr": 2.283, "cost_per_test": 3.074655, "token_totals": { "input_tokens": 410193956, "output_tokens": 9698383, "reasoning_tokens": 7045672, "cache_read_tokens": 393497338, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 71.088, "latency": 1107.811, "stderr": 2.748, "cost_per_test": 1.143199, "token_totals": { "input_tokens": 129631093, "output_tokens": 6056928, "reasoning_tokens": 3495567, "cache_read_tokens": 112922114, "cache_write_tokens": 16703171 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 69.954, "latency": 1041.052, "stderr": 2.542, "cost_per_test": 0.141708, "token_totals": { "input_tokens": 362285754, "output_tokens": 11007243, "reasoning_tokens": 8590626, "cache_read_tokens": 332678280, "cache_write_tokens": 29598418 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 69.552, "latency": 2615.096, "stderr": 2.486, "cost_per_test": 1.329318, "token_totals": { "input_tokens": 1262287639, "output_tokens": 14703425, "reasoning_tokens": 0, "cache_read_tokens": 1227785728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 69.173, "latency": 1971.687, "stderr": 2.395, "cost_per_test": 0.330922, "token_totals": { "input_tokens": 1809205546, "output_tokens": 16117195, "reasoning_tokens": 11510477, "cache_read_tokens": 1797102837, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 69.112, "latency": 2172.231, "stderr": 2.649, "cost_per_test": 3.787827, "token_totals": { "input_tokens": 942177449, "output_tokens": 18177772, "reasoning_tokens": null, "cache_read_tokens": 891631657, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 68.972, "latency": 4904.981, "stderr": 2.339, "cost_per_test": 4.839324, "token_totals": { "input_tokens": 766628987, "output_tokens": 17347923, "reasoning_tokens": null, "cache_read_tokens": 509441536, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 68.946, "latency": 1839.41, "stderr": 2.402, "cost_per_test": 0.787396, "token_totals": { "input_tokens": 824410363, "output_tokens": 10431429, "reasoning_tokens": 7444891, "cache_read_tokens": 799662720, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 68.882, "latency": 1521.906, "stderr": 2.267, "cost_per_test": 0.108634, "token_totals": { "input_tokens": 891374421, "output_tokens": 16056047, "reasoning_tokens": 12655956, "cache_read_tokens": 861570496, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 67.843, "latency": 1847.78, "stderr": 2.29, "cost_per_test": 0.268958, "token_totals": { "input_tokens": 1127000678, "output_tokens": 14186544, "reasoning_tokens": 10606514, "cache_read_tokens": 1112061687, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 67.758, "latency": 780.458, "stderr": 2.565, "cost_per_test": 0.234152, "token_totals": { "input_tokens": 893420650, "output_tokens": 13279281, "reasoning_tokens": 10184478, "cache_read_tokens": 884618493, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 67.347, "latency": 3313.924, "stderr": 2.903, "cost_per_test": 0.328999, "token_totals": { "input_tokens": 847310790, "output_tokens": 17554862, "reasoning_tokens": 14594001, "cache_read_tokens": 813521152, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 66.252, "latency": 4495.202, "stderr": 2.685, "cost_per_test": 0.713033, "token_totals": { "input_tokens": 3932720612, "output_tokens": 29249606, "reasoning_tokens": null, "cache_read_tokens": 3849386240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 65.477, "latency": 1195.527, "stderr": 3.691, "cost_per_test": 1.364559, "token_totals": { "input_tokens": 1482730908, "output_tokens": 25203657, "reasoning_tokens": 19717258, "cache_read_tokens": 1444841999, "cache_write_tokens": 37877007 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 65.128, "latency": 2247.236, "stderr": 2.35, "cost_per_test": 1.08141, "token_totals": { "input_tokens": 864870937, "output_tokens": 15254147, "reasoning_tokens": 11469965, "cache_read_tokens": 855589504, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "reflection/Beam-1.0": { "accuracy": 64.733, "latency": 1204.28, "stderr": 2.259, "cost_per_test": null, "token_totals": { "input_tokens": 905656608, "output_tokens": 6481191, "reasoning_tokens": 4044707, "cache_read_tokens": 884618112, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "meta/muse-spark-open": { "accuracy": 61.983, "latency": 5078.614, "stderr": 2.704, "cost_per_test": 5.348821, "token_totals": { "input_tokens": 973042320, "output_tokens": 8495845, "reasoning_tokens": 4888586, "cache_read_tokens": 642573184, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 59.754, "latency": 1298.806, "stderr": 2.537, "cost_per_test": 3.18384, "token_totals": { "input_tokens": 2167916385, "output_tokens": 14444692, "reasoning_tokens": 4485341, "cache_read_tokens": 2146906887, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 40.971, "latency": 1581.835, "stderr": 2.982, "cost_per_test": 1.671783, "token_totals": { "input_tokens": 658436029, "output_tokens": 9281665, "reasoning_tokens": null, "cache_read_tokens": 641195264, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 40.823, "latency": 1173.625, "stderr": 3.21, "cost_per_test": 0.249301, "token_totals": { "input_tokens": 254752442, "output_tokens": 6362193, "reasoning_tokens": null, "cache_read_tokens": 245337472, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "tax_agent_bench": { "anthropic/claude-opus-5-5": { "accuracy": 45.573, "latency": 4012.548, "stderr": 3.593, "cost_per_test": 15.086184, "token_totals": { "input_tokens": 815304119, "output_tokens": 88000495, "reasoning_tokens": 76703031, "cache_read_tokens": 648981098, "cache_write_tokens": 166280701 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 44.56, "latency": 985.634, "stderr": 3.587, "cost_per_test": 3.473645, "token_totals": { "input_tokens": 285519334, "output_tokens": 14493886, "reasoning_tokens": 6146392, "cache_read_tokens": 200405700, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 44.35, "latency": 1264.434, "stderr": 3.574, "cost_per_test": 0.459454, "token_totals": { "input_tokens": 779160448, "output_tokens": 53561725, "reasoning_tokens": 45716077, "cache_read_tokens": 712066688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 42.997, "latency": 1084.296, "stderr": 3.572, "cost_per_test": 2.147235, "token_totals": { "input_tokens": 257606995, "output_tokens": 9968530, "reasoning_tokens": 6086406, "cache_read_tokens": 214574250, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 41.959, "latency": 3560.542, "stderr": 3.561, "cost_per_test": 9.245466, "token_totals": { "input_tokens": 1150865417, "output_tokens": 108029684, "reasoning_tokens": 94199819, "cache_read_tokens": 954147139, "cache_write_tokens": 196666034 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 39.698, "latency": 3052.932, "stderr": 3.52, "cost_per_test": 1.80097, "token_totals": { "input_tokens": 229302538, "output_tokens": 47417212, "reasoning_tokens": 1131, "cache_read_tokens": 154199490, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-wonder": { "accuracy": 36.269, "latency": 492.843, "stderr": 3.47, "cost_per_test": 1.884439, "token_totals": { "input_tokens": 225535207, "output_tokens": 8770631, "reasoning_tokens": 5466021, "cache_read_tokens": 187857044, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 34.197, "latency": 996.469, "stderr": 3.423, "cost_per_test": 1.796539, "token_totals": { "input_tokens": 280409764, "output_tokens": 10821528, "reasoning_tokens": 6708987, "cache_read_tokens": 186011136, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 33.368, "latency": 2415.159, "stderr": 3.386, "cost_per_test": 2.790127, "token_totals": { "input_tokens": 444117224, "output_tokens": 14995598, "reasoning_tokens": 10717157, "cache_read_tokens": 381463574, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 32.617, "latency": 2464.028, "stderr": 3.381, "cost_per_test": 0.585277, "token_totals": { "input_tokens": 181603577, "output_tokens": 28903691, "reasoning_tokens": 24008954, "cache_read_tokens": 139882560, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 32.388, "latency": 1547.695, "stderr": 3.361, "cost_per_test": 0.134306, "token_totals": { "input_tokens": 136588210, "output_tokens": 12521886, "reasoning_tokens": 8284595, "cache_read_tokens": 102894848, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 32.136, "latency": 1962.771, "stderr": 3.342, "cost_per_test": 0.320948, "token_totals": { "input_tokens": 551883325, "output_tokens": 70312257, "reasoning_tokens": 62154212, "cache_read_tokens": 466865454, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 32.124, "latency": 178.329, "stderr": 3.37, "cost_per_test": 0.857214, "token_totals": { "input_tokens": 262586790, "output_tokens": 7012701, "reasoning_tokens": 4789793, "cache_read_tokens": 208172664, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 31.922, "latency": 2973.647, "stderr": 3.352, "cost_per_test": 1.164207, "token_totals": { "input_tokens": 782193467, "output_tokens": 143162719, "reasoning_tokens": 133330984, "cache_read_tokens": 613966554, "cache_write_tokens": 168173092 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 31.55, "latency": 1297.006, "stderr": 3.35, "cost_per_test": 0.138062, "token_totals": { "input_tokens": 194143568, "output_tokens": 28577937, "reasoning_tokens": 24325230, "cache_read_tokens": 139704320, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 30.391, "latency": 3438.427, "stderr": 3.309, "cost_per_test": 2.72124, "token_totals": { "input_tokens": 561529891, "output_tokens": 26978904, "reasoning_tokens": null, "cache_read_tokens": 287912448, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 28.507, "latency": 1038.928, "stderr": 3.14, "cost_per_test": 0.048982, "token_totals": { "input_tokens": 168296091, "output_tokens": 12132305, "reasoning_tokens": 7352234, "cache_read_tokens": 127944128, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 27.273, "latency": 1288.515, "stderr": 3.113, "cost_per_test": 0.55338, "token_totals": { "input_tokens": 311523716, "output_tokens": 16401077, "reasoning_tokens": 0, "cache_read_tokens": 262109696, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 26.756, "latency": 1596.755, "stderr": 3.093, "cost_per_test": 0.729136, "token_totals": { "input_tokens": 172298347, "output_tokens": 21059828, "reasoning_tokens": 17608474, "cache_read_tokens": 133313152, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 26.63, "latency": 349.076, "stderr": 3.141, "cost_per_test": 0.134662, "token_totals": { "input_tokens": 114496920, "output_tokens": 15566679, "reasoning_tokens": 12729723, "cache_read_tokens": 91970688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 26.321, "latency": 701.825, "stderr": 2.653, "cost_per_test": 0.883559, "token_totals": { "input_tokens": 413690950, "output_tokens": 3571385, "reasoning_tokens": 1667757, "cache_read_tokens": 329680688, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 21.754, "latency": 1213.878, "stderr": 2.977, "cost_per_test": 1.323601, "token_totals": { "input_tokens": 149308416, "output_tokens": 12717430, "reasoning_tokens": 9488118, "cache_read_tokens": 103439338, "cache_write_tokens": 45805894 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.717, "latency": 900.586, "stderr": 2.924, "cost_per_test": 5.7963, "token_totals": { "input_tokens": 115812555, "output_tokens": 12032734, "reasoning_tokens": 9165100, "cache_read_tokens": 71528751, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.588, "latency": 314.575, "stderr": 2.874, "cost_per_test": 0.164063, "token_totals": { "input_tokens": 85169988, "output_tokens": 4820320, "reasoning_tokens": 176867, "cache_read_tokens": 22014292, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 19.947, "latency": 1995.736, "stderr": 2.866, "cost_per_test": 0.190956, "token_totals": { "input_tokens": 449103207, "output_tokens": 40664272, "reasoning_tokens": 37199155, "cache_read_tokens": 368052701, "cache_write_tokens": 80918209 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 15.389, "latency": 385.252, "stderr": 2.365, "cost_per_test": 0.091767, "token_totals": { "input_tokens": 107926904, "output_tokens": 7430357, "reasoning_tokens": null, "cache_read_tokens": 98264704, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 15.291, "latency": 810.557, "stderr": 2.423, "cost_per_test": 0.353757, "token_totals": { "input_tokens": 147459631, "output_tokens": 7269937, "reasoning_tokens": null, "cache_read_tokens": 130876928, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "reflection/Beam-1.0": { "accuracy": 8.808, "latency": 57.987, "stderr": 1.902, "cost_per_test": null, "token_totals": { "input_tokens": 110261613, "output_tokens": 4023578, "reasoning_tokens": 2422138, "cache_read_tokens": 88299456, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null } } } }