{ "metadata": { "benchmark": "EMB", "slug": "emb", "description": "Evaluating agents on Excel-based financial modeling tasks", "benchmark_id": "emb", "family": "emb", "version": "1", "updated": "2026-10-20", "dataset_type": "private", "industry": "finance", "tasks": { "overall": "Overall", "template": "Template", "scratch": "Scratch", "lbo": "LBO", "dcf": "DCF", "ma": "M&A", "operating": "Operating", "comps": "Comps", "dataroom": "Dataroom", "misc": "Misc" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "alibaba/qwen3.8-max-internal", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "fireworks/ember-1", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-large-4", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": true, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 77 }, "tasks": { "overall": { "anthropic/claude-fable-5-1": { "accuracy": 76.67, "latency": 1771.558, "stderr": 2.085, "cost_per_test": 15.93249, "token_totals": { "input_tokens": 454757375, "output_tokens": 12499169, "reasoning_tokens": 8150917, "cache_read_tokens": 378586242, "cache_write_tokens": 76164267 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.938, "latency": 2106.434, "stderr": 2.383, "cost_per_test": 11.018974, "token_totals": { "input_tokens": 922243024, "output_tokens": 23704546, "reasoning_tokens": 18569025, "cache_read_tokens": 861165257, "cache_write_tokens": 61067293 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.713, "latency": 2204.991, "stderr": 2.437, "cost_per_test": 8.792863, "token_totals": { "input_tokens": 2367951142, "output_tokens": 29328538, "reasoning_tokens": 22320203, "cache_read_tokens": 2311427347, "cache_write_tokens": 56507215 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 75.244, "latency": 2305.821, "stderr": 2.459, "cost_per_test": 11.8029, "token_totals": { "input_tokens": 1127460088, "output_tokens": 14257556, "reasoning_tokens": 9497334, "cache_read_tokens": 945025185, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 73.666, "latency": 1594.153, "stderr": 2.468, "cost_per_test": 12.354419, "token_totals": { "input_tokens": 345067688, "output_tokens": 11953058, "reasoning_tokens": null, "cache_read_tokens": 318538087, "cache_write_tokens": 26523742 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 73.556, "latency": 1558.673, "stderr": 2.24, "cost_per_test": 6.071566, "token_totals": { "input_tokens": 686017503, "output_tokens": 12323715, "reasoning_tokens": 7957163, "cache_read_tokens": 656949908, "cache_write_tokens": 29058427 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 72.339, "latency": 590.258, "stderr": 2.373, "cost_per_test": 6.010773, "token_totals": { "input_tokens": 318866006, "output_tokens": 5457571, "reasoning_tokens": 3316634, "cache_read_tokens": 276762112, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 72.196, "latency": 769.197, "stderr": 2.42, "cost_per_test": 8.243862, "token_totals": { "input_tokens": 2960728904, "output_tokens": 21722815, "reasoning_tokens": 13162823, "cache_read_tokens": 2787511478, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 71.979, "latency": 3193.384, "stderr": 2.387, "cost_per_test": 4.430287, "token_totals": { "input_tokens": 1260695298, "output_tokens": 14484385, "reasoning_tokens": 10207311, "cache_read_tokens": 1232232960, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-wonder-full": { "accuracy": 71.917, "latency": 1684.504, "stderr": 2.658, "cost_per_test": 4.58223, "token_totals": { "input_tokens": 983252171, "output_tokens": 8330149, "reasoning_tokens": 5099887, "cache_read_tokens": 955848482, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 71.7, "latency": 804.28, "stderr": 2.475, "cost_per_test": 5.827063, "token_totals": { "input_tokens": 123989946, "output_tokens": 5542631, "reasoning_tokens": 3051513, "cache_read_tokens": 104449111, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 71.53, "latency": 621.736, "stderr": 2.351, "cost_per_test": 1.276882, "token_totals": { "input_tokens": 216104819, "output_tokens": 5241605, "reasoning_tokens": 3222968, "cache_read_tokens": 204982294, "cache_write_tokens": 11114719 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 71.326, "latency": 650.914, "stderr": 2.262, "cost_per_test": 5.694001, "token_totals": { "input_tokens": 1686304951, "output_tokens": 18606388, "reasoning_tokens": 7782800, "cache_read_tokens": 1546827567, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 70.773, "latency": 1107.811, "stderr": 2.712, "cost_per_test": 1.143199, "token_totals": { "input_tokens": 129631093, "output_tokens": 6056928, "reasoning_tokens": 3495567, "cache_read_tokens": 112922114, "cache_write_tokens": 16703171 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 69.373, "latency": 2632.285, "stderr": 2.606, "cost_per_test": 12.05844, "token_totals": { "input_tokens": 833619980, "output_tokens": 22136588, "reasoning_tokens": null, "cache_read_tokens": 790543547, "cache_write_tokens": 43067000 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 68.515, "latency": 1041.052, "stderr": 2.809, "cost_per_test": 0.141708, "token_totals": { "input_tokens": 362285754, "output_tokens": 11007243, "reasoning_tokens": 8590626, "cache_read_tokens": 332678280, "cache_write_tokens": 29598418 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 67.426, "latency": 829.829, "stderr": 3.056, "cost_per_test": 2.546868, "token_totals": { "input_tokens": 1115802604, "output_tokens": 13342216, "reasoning_tokens": 8360823, "cache_read_tokens": 1083342803, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 67.116, "latency": 660.047, "stderr": 2.936, "cost_per_test": 0.366502, "token_totals": { "input_tokens": 653086981, "output_tokens": 6955414, "reasoning_tokens": 4412805, "cache_read_tokens": 604381577, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 67.084, "latency": 1034.437, "stderr": 2.891, "cost_per_test": 5.599337, "token_totals": { "input_tokens": 1072958990, "output_tokens": 8128857, "reasoning_tokens": 5046443, "cache_read_tokens": 1021915980, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 66.991, "latency": 2097.023, "stderr": 3.045, "cost_per_test": 6.481284, "token_totals": { "input_tokens": 531961928, "output_tokens": 15529729, "reasoning_tokens": 10549516, "cache_read_tokens": 448911360, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.677, "latency": 741.42, "stderr": 2.873, "cost_per_test": 3.074655, "token_totals": { "input_tokens": 410193956, "output_tokens": 9698383, "reasoning_tokens": 7045672, "cache_read_tokens": 393497338, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 66.316, "latency": 2985.704, "stderr": 3.006, "cost_per_test": 10.293794, "token_totals": { "input_tokens": 3099396585, "output_tokens": 25999381, "reasoning_tokens": null, "cache_read_tokens": 3034388542, "cache_write_tokens": 64983558 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.205, "latency": 813.024, "stderr": 3.11, "cost_per_test": 2.193037, "token_totals": { "input_tokens": 314065634, "output_tokens": 7024530, "reasoning_tokens": 4854711, "cache_read_tokens": 286789081, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 65.459, "latency": 1521.906, "stderr": 2.709, "cost_per_test": 0.108634, "token_totals": { "input_tokens": 891374421, "output_tokens": 16056047, "reasoning_tokens": 12655956, "cache_read_tokens": 861570496, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 65.411, "latency": 726.48, "stderr": 2.646, "cost_per_test": 3.893069, "token_totals": { "input_tokens": 1058720610, "output_tokens": 15239728, "reasoning_tokens": 6777930, "cache_read_tokens": 972645922, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 64.543, "latency": 910.575, "stderr": 2.869, "cost_per_test": 3.273352, "token_totals": { "input_tokens": 183023742, "output_tokens": 5181007, "reasoning_tokens": 3094376, "cache_read_tokens": 167235072, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 63.728, "latency": 2315.595, "stderr": 3.106, "cost_per_test": 12.756352, "token_totals": { "input_tokens": 1214546122, "output_tokens": 18458044, "reasoning_tokens": 10154511, "cache_read_tokens": 1174122388, "cache_write_tokens": 40413529 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 63.548, "latency": 1123.555, "stderr": 2.752, "cost_per_test": 5.173115, "token_totals": { "input_tokens": 1617865425, "output_tokens": 16984726, "reasoning_tokens": 10956816, "cache_read_tokens": 1520002205, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 63.368, "latency": 1867.313, "stderr": 2.949, "cost_per_test": 2.459539, "token_totals": { "input_tokens": 287599624, "output_tokens": 7246798, "reasoning_tokens": 4945223, "cache_read_tokens": 266899209, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 62.858, "latency": 3313.924, "stderr": 3.141, "cost_per_test": 0.328999, "token_totals": { "input_tokens": 847310790, "output_tokens": 17554862, "reasoning_tokens": 14594001, "cache_read_tokens": 813521152, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.6": { "accuracy": 62.727, "latency": 3178.533, "stderr": 3.085, "cost_per_test": 3.056382, "token_totals": { "input_tokens": 292867165, "output_tokens": 18238122, "reasoning_tokens": 12738109, "cache_read_tokens": 269312000, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 62.712, "latency": 1374.999, "stderr": 2.956, "cost_per_test": 3.552349, "token_totals": { "input_tokens": 1152349029, "output_tokens": 12397789, "reasoning_tokens": 7178696, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 61.805, "latency": 1007.461, "stderr": 3.192, "cost_per_test": 0.355142, "token_totals": { "input_tokens": 2089267477, "output_tokens": 16658390, "reasoning_tokens": 11880969, "cache_read_tokens": 2076686592, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 61.532, "latency": 3532.407, "stderr": 2.879, "cost_per_test": 4.465213, "token_totals": { "input_tokens": 1022219898, "output_tokens": 19460303, "reasoning_tokens": 14618756, "cache_read_tokens": 930948672, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.154, "latency": 3255.853, "stderr": 3.276, "cost_per_test": 7.22787, "token_totals": { "input_tokens": 898138518, "output_tokens": 21113414, "reasoning_tokens": null, "cache_read_tokens": 854339701, "cache_write_tokens": 43790957 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 60.066, "latency": 3467.209, "stderr": 3.352, "cost_per_test": 2.655924, "token_totals": { "input_tokens": 599112401, "output_tokens": 12051178, "reasoning_tokens": 8874470, "cache_read_tokens": 572733440, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 59.661, "latency": 1370.794, "stderr": 3.029, "cost_per_test": 4.556437, "token_totals": { "input_tokens": 821326544, "output_tokens": 18031108, "reasoning_tokens": 13367587, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 59.015, "latency": 1082.902, "stderr": 4.17, "cost_per_test": 1.146892, "token_totals": { "input_tokens": 1240305362, "output_tokens": 21428591, "reasoning_tokens": 16654436, "cache_read_tokens": 1207542704, "cache_write_tokens": 32752108 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 58.076, "latency": 1839.41, "stderr": 3.136, "cost_per_test": 0.787396, "token_totals": { "input_tokens": 824410363, "output_tokens": 10431429, "reasoning_tokens": 7444891, "cache_read_tokens": 799662720, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 57.855, "latency": 2387.544, "stderr": 2.739, "cost_per_test": 2.195977, "token_totals": { "input_tokens": 861752101, "output_tokens": 14639278, "reasoning_tokens": 0, "cache_read_tokens": 826875904, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 57.208, "latency": 780.458, "stderr": 3.227, "cost_per_test": 0.234152, "token_totals": { "input_tokens": 893420650, "output_tokens": 13279281, "reasoning_tokens": 10184478, "cache_read_tokens": 884618493, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 57.116, "latency": 2615.096, "stderr": 3.165, "cost_per_test": 1.329318, "token_totals": { "input_tokens": 1262287639, "output_tokens": 14703425, "reasoning_tokens": 0, "cache_read_tokens": 1227785728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 57.041, "latency": 1957.165, "stderr": 2.853, "cost_per_test": 2.857267, "token_totals": { "input_tokens": 275851986, "output_tokens": 9259862, "reasoning_tokens": 4794212, "cache_read_tokens": 207838976, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 56.978, "latency": 1027.476, "stderr": 3.127, "cost_per_test": 2.313636, "token_totals": { "input_tokens": 829404547, "output_tokens": 10741088, "reasoning_tokens": 5180580, "cache_read_tokens": 769467634, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 56.977, "latency": 1574.097, "stderr": 3.391, "cost_per_test": 0.383709, "token_totals": { "input_tokens": 1018385162, "output_tokens": 15135599, "reasoning_tokens": 11153857, "cache_read_tokens": 1008679680, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 56.361, "latency": 891.512, "stderr": 3.153, "cost_per_test": 1.112498, "token_totals": { "input_tokens": 368051977, "output_tokens": 9069516, "reasoning_tokens": 4627614, "cache_read_tokens": 350123312, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 56.344, "latency": 2172.231, "stderr": 3.319, "cost_per_test": 3.787827, "token_totals": { "input_tokens": 942177449, "output_tokens": 18177772, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 56.213, "latency": 4904.981, "stderr": 3.22, "cost_per_test": 4.839324, "token_totals": { "input_tokens": 766628987, "output_tokens": 17347923, "reasoning_tokens": null, "cache_read_tokens": 509441536, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 55.932, "latency": 4495.202, "stderr": 3.561, "cost_per_test": 0.713033, "token_totals": { "input_tokens": 3932720612, "output_tokens": 29249606, "reasoning_tokens": null, "cache_read_tokens": 3849386240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 55.228, "latency": 1747.98, "stderr": 3.112, "cost_per_test": 0.221264, "token_totals": { "input_tokens": 455429765, "output_tokens": 11928848, "reasoning_tokens": 6975136, "cache_read_tokens": 430971584, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 55.092, "latency": 1837.949, "stderr": 3.127, "cost_per_test": 0.083304, "token_totals": { "input_tokens": 599516539, "output_tokens": 12515553, "reasoning_tokens": 6917639, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-spark-open": { "accuracy": 53.667, "latency": 5078.614, "stderr": 3.196, "cost_per_test": 5.348821, "token_totals": { "input_tokens": 973042320, "output_tokens": 8495845, "reasoning_tokens": 4888586, "cache_read_tokens": 642573184, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4.5": { "accuracy": 52.911, "latency": 1107.838, "stderr": 3.111, "cost_per_test": 1.55492, "token_totals": { "input_tokens": 150960387, "output_tokens": 7436273, "reasoning_tokens": 2117149, "cache_read_tokens": 129340672, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.803, "latency": 2247.236, "stderr": 3.059, "cost_per_test": 1.08141, "token_totals": { "input_tokens": 864870937, "output_tokens": 15254147, "reasoning_tokens": 11469965, "cache_read_tokens": 855589504, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 52.616, "latency": 708.582, "stderr": 2.974, "cost_per_test": 3.769246, "token_totals": { "input_tokens": 703037870, "output_tokens": 5442617, "reasoning_tokens": 2522069, "cache_read_tokens": 656983985, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 51.623, "latency": 822.144, "stderr": 3.053, "cost_per_test": 0.796757, "token_totals": { "input_tokens": 541981728, "output_tokens": 9803133, "reasoning_tokens": 3184172, "cache_read_tokens": 527403648, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 49.345, "latency": 2209.243, "stderr": 3.181, "cost_per_test": 0.597724, "token_totals": { "input_tokens": 414159309, "output_tokens": 10957606, "reasoning_tokens": 5555265, "cache_read_tokens": 355632384, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 47.759, "latency": 1890.343, "stderr": 2.946, "cost_per_test": 2.091762, "token_totals": { "input_tokens": 1398807070, "output_tokens": 13708644, "reasoning_tokens": null, "cache_read_tokens": 1372552733, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.426, "latency": 2601.215, "stderr": 3.645, "cost_per_test": 2.146984, "token_totals": { "input_tokens": 688505179, "output_tokens": 26360621, "reasoning_tokens": 22515833, "cache_read_tokens": 616310016, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 44.751, "latency": 1767.043, "stderr": 3.121, "cost_per_test": 0.57312, "token_totals": { "input_tokens": 1149607508, "output_tokens": 16413434, "reasoning_tokens": 10666857, "cache_read_tokens": 1066555648, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 43.231, "latency": 493.632, "stderr": 3.105, "cost_per_test": 1.231783, "token_totals": { "input_tokens": 1498941214, "output_tokens": 10717779, "reasoning_tokens": 4101020, "cache_read_tokens": 1299388807, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 38.362, "latency": 1581.835, "stderr": 3.273, "cost_per_test": 1.671783, "token_totals": { "input_tokens": 658436029, "output_tokens": 9281665, "reasoning_tokens": null, "cache_read_tokens": 641195264, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 36.832, "latency": 2207.467, "stderr": 3.325, "cost_per_test": 0.179543, "token_totals": { "input_tokens": 952446737, "output_tokens": 16573128, "reasoning_tokens": null, "cache_read_tokens": 886140288, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 32.867, "latency": 1873.595, "stderr": 2.814, "cost_per_test": 2.866775, "token_totals": { "input_tokens": 521107748, "output_tokens": 8556725, "reasoning_tokens": 1171101, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 31.762, "latency": 1173.625, "stderr": 2.956, "cost_per_test": 0.249301, "token_totals": { "input_tokens": 254752442, "output_tokens": 6362193, "reasoning_tokens": null, "cache_read_tokens": 245337472, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 31.735, "latency": 2382.355, "stderr": 2.805, "cost_per_test": null, "token_totals": { "input_tokens": 591493477, "output_tokens": 12813242, "reasoning_tokens": null, "cache_read_tokens": 153934848, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 28.469, "latency": 3796.855, "stderr": 2.23, "cost_per_test": 0.786639, "token_totals": { "input_tokens": 515628666, "output_tokens": 6819932, "reasoning_tokens": 689099, "cache_read_tokens": 500772956, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 28.451, "latency": 1207.259, "stderr": 2.431, "cost_per_test": 1.658686, "token_totals": { "input_tokens": 547048102, "output_tokens": 8578062, "reasoning_tokens": 1143349, "cache_read_tokens": 26409876, "cache_write_tokens": 3330859 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 26.017, "latency": 328.69, "stderr": 2.912, "cost_per_test": 0.453183, "token_totals": { "input_tokens": 252451404, "output_tokens": 4842419, "reasoning_tokens": 1535428, "cache_read_tokens": 210062842, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 25.006, "latency": 735.485, "stderr": 2.462, "cost_per_test": 0.179925, "token_totals": { "input_tokens": 486672516, "output_tokens": 18585242, "reasoning_tokens": null, "cache_read_tokens": 379610210, "cache_write_tokens": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 22.664, "latency": 718.845, "stderr": 2.392, "cost_per_test": 1.030784, "token_totals": { "input_tokens": 426603491, "output_tokens": 8226592, "reasoning_tokens": 1108260, "cache_read_tokens": 408024356, "cache_write_tokens": 18501707 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 18.394, "latency": 271.547, "stderr": 1.667, "cost_per_test": 0.634124, "token_totals": { "input_tokens": 139153568, "output_tokens": 4590085, "reasoning_tokens": 340343, "cache_read_tokens": 118693351, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 16.987, "latency": 2359.322, "stderr": 1.452, "cost_per_test": 0.182153, "token_totals": { "input_tokens": 1042500185, "output_tokens": 30098913, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 11.739, "latency": 367.615, "stderr": 1.553, "cost_per_test": 0.469926, "token_totals": { "input_tokens": 82567089, "output_tokens": 1859488, "reasoning_tokens": 251920, "cache_read_tokens": 71571428, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 11.185, "latency": 3082.124, "stderr": 1.637, "cost_per_test": 10.603072, "token_totals": { "input_tokens": 428119963, "output_tokens": 6269299, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 8.789, "latency": 300.757, "stderr": 1.275, "cost_per_test": 0.417608, "token_totals": { "input_tokens": 407236592, "output_tokens": 5531340, "reasoning_tokens": 5399594, "cache_read_tokens": 238888010, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 8.629, "latency": 67.007, "stderr": 1.088, "cost_per_test": 0.069424, "token_totals": { "input_tokens": 77708032, "output_tokens": 1187254, "reasoning_tokens": 588837, "cache_read_tokens": 62784855, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "template": { "anthropic/claude-opus-5-5": { "accuracy": 81.385, "latency": 1928.473, "stderr": 2.545, "cost_per_test": 10.771299, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 80.939, "latency": 2158.229, "stderr": 2.616, "cost_per_test": 9.010402, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 80.796, "latency": 1699.436, "stderr": 2.54, "cost_per_test": 16.559081, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 79.553, "latency": 2452.597, "stderr": 2.887, "cost_per_test": 16.142633, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 76.991, "latency": 1549.904, "stderr": 2.631, "cost_per_test": 12.636539, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 75.58, "latency": 813.595, "stderr": 3.201, "cost_per_test": 9.502756, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 75.412, "latency": 3263.425, "stderr": 2.864, "cost_per_test": 4.88682, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 75.303, "latency": 558.195, "stderr": 3.068, "cost_per_test": 7.895, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 75.072, "latency": 959.915, "stderr": 3.311, "cost_per_test": 0.996387, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 74.785, "latency": 721.493, "stderr": 3.343, "cost_per_test": 5.767376, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 74.693, "latency": 1494.85, "stderr": 3.393, "cost_per_test": 6.50608, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 73.601, "latency": 577.305, "stderr": 3.437, "cost_per_test": 6.069648, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 73.488, "latency": 555.706, "stderr": 3.378, "cost_per_test": 1.329992, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 73.325, "latency": 1834.408, "stderr": 3.997, "cost_per_test": 5.314337, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 71.143, "latency": 2724.158, "stderr": 3.696, "cost_per_test": 12.960012, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 70.405, "latency": 1085.666, "stderr": 3.702, "cost_per_test": 0.174611, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 68.49, "latency": 672.659, "stderr": 4.047, "cost_per_test": 2.258682, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 67.983, "latency": 2843.855, "stderr": 4.194, "cost_per_test": 10.117958, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.69, "latency": 713.238, "stderr": 4.428, "cost_per_test": 3.107736, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder": { "accuracy": 65.731, "latency": 1119.036, "stderr": 4.795, "cost_per_test": 7.413135, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 65.649, "latency": 1842.178, "stderr": 4.502, "cost_per_test": 6.713146, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 65.54, "latency": 865.494, "stderr": 4.601, "cost_per_test": 2.983089, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 64.902, "latency": 3392.353, "stderr": 4.545, "cost_per_test": 0.368856, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 64.747, "latency": 634.713, "stderr": 4.193, "cost_per_test": 0.491324, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 64.747, "latency": 698.325, "stderr": 3.843, "cost_per_test": 4.247264, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 64.555, "latency": 1572.313, "stderr": 4.288, "cost_per_test": 0.117057, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 64.253, "latency": 1031.123, "stderr": 5.345, "cost_per_test": 1.118607, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 62.531, "latency": 1171.71, "stderr": 4.267, "cost_per_test": 5.601891, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 61.809, "latency": 873.526, "stderr": 4.311, "cost_per_test": 3.134312, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 60.812, "latency": 1867.103, "stderr": 5.002, "cost_per_test": 2.757625, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 60.423, "latency": 2326.254, "stderr": 4.558, "cost_per_test": 13.128173, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 59.212, "latency": 1451.63, "stderr": 4.569, "cost_per_test": 4.55582, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.2": { "accuracy": 55.814, "latency": 3556.073, "stderr": 4.771, "cost_per_test": 4.902211, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 55.683, "latency": 2346.955, "stderr": 3.761, "cost_per_test": 2.237057, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 55.17, "latency": 3118.875, "stderr": 4.807, "cost_per_test": 6.851896, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 55.108, "latency": 3340.715, "stderr": 5.111, "cost_per_test": 2.693945, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 54.729, "latency": 3258.312, "stderr": 5.331, "cost_per_test": 3.19122, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 54.684, "latency": 1462.328, "stderr": 4.693, "cost_per_test": 4.762024, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 54.659, "latency": 1003.776, "stderr": 5.194, "cost_per_test": 0.362471, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 52.237, "latency": 1830.734, "stderr": 4.375, "cost_per_test": 0.080822, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 52.167, "latency": 1608.941, "stderr": 4.601, "cost_per_test": 0.410332, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 52.116, "latency": 1742.23, "stderr": 4.677, "cost_per_test": 0.231587, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 51.787, "latency": 1951.777, "stderr": 4.788, "cost_per_test": 2.970414, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 49.917, "latency": 1084.275, "stderr": 4.957, "cost_per_test": 2.878016, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 49.41, "latency": 914.94, "stderr": 5.188, "cost_per_test": 1.209257, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 49.106, "latency": 3937.654, "stderr": 5.381, "cost_per_test": 0.497756, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 48.717, "latency": 1835.197, "stderr": 5.083, "cost_per_test": 0.830004, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 48.181, "latency": 845.999, "stderr": 5.238, "cost_per_test": 0.257032, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 48.123, "latency": 5580.684, "stderr": 5.327, "cost_per_test": 5.810682, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 47.417, "latency": 2723.319, "stderr": 5.427, "cost_per_test": 1.44076, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 47.213, "latency": 792.971, "stderr": 4.677, "cost_per_test": 4.390484, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 46.07, "latency": 777.796, "stderr": 4.377, "cost_per_test": 0.799253, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 46.053, "latency": 2207.215, "stderr": 5.563, "cost_per_test": 3.97223, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 45.052, "latency": 2026.409, "stderr": 4.661, "cost_per_test": 0.661814, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 44.632, "latency": 5739.054, "stderr": 4.971, "cost_per_test": 7.009169, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 41.498, "latency": 2071.822, "stderr": 5.26, "cost_per_test": 0.54275, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 40.906, "latency": 509.542, "stderr": 4.556, "cost_per_test": 1.314071, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 39.291, "latency": 2234.018, "stderr": 5.18, "cost_per_test": 1.117392, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 38.301, "latency": 1281.579, "stderr": 5.486, "cost_per_test": 1.605523, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 36.014, "latency": 2668.546, "stderr": 4.913, "cost_per_test": 2.283165, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 34.546, "latency": 1804.175, "stderr": 4.741, "cost_per_test": 2.328648, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 32.521, "latency": 1611.127, "stderr": 4.575, "cost_per_test": 1.841659, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 23.292, "latency": 2150.339, "stderr": 4.44, "cost_per_test": 0.180096, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 21.904, "latency": 1682.503, "stderr": 4.345, "cost_per_test": 2.697909, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 21.417, "latency": 718.508, "stderr": 3.554, "cost_per_test": 0.185054, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 20.769, "latency": 355.336, "stderr": 4.223, "cost_per_test": 0.5034, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 19.798, "latency": 1286.708, "stderr": 3.487, "cost_per_test": 1.780973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 18.647, "latency": 1984.68, "stderr": 3.826, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 18.439, "latency": 3535.258, "stderr": 3.233, "cost_per_test": 0.702023, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 18.202, "latency": 1174.638, "stderr": 4.0, "cost_per_test": 0.278353, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 16.42, "latency": 734.223, "stderr": 3.262, "cost_per_test": 1.009601, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 15.993, "latency": 2634.339, "stderr": 2.465, "cost_per_test": 0.18636, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 8.775, "latency": 426.285, "stderr": 2.181, "cost_per_test": 0.51072, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 8.137, "latency": 70.993, "stderr": 1.799, "cost_per_test": 0.074608, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 7.386, "latency": 2964.222, "stderr": 2.262, "cost_per_test": 10.352195, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.3": { "accuracy": 6.847, "latency": 261.611, "stderr": 2.313, "cost_per_test": 0.713911, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.833, "latency": 326.512, "stderr": 1.652, "cost_per_test": 0.429915, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "scratch": { "anthropic/claude-fable-5-1": { "accuracy": 72.593, "latency": 1843.681, "stderr": 2.339, "cost_per_test": 15.305899, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 72.255, "latency": 1622.496, "stderr": 2.573, "cost_per_test": 5.637053, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 70.798, "latency": 2159.046, "stderr": 2.687, "cost_per_test": 7.463166, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 70.792, "latency": 3098.755, "stderr": 2.503, "cost_per_test": 2.921544, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 70.547, "latency": 1534.599, "stderr": 2.616, "cost_per_test": 3.850123, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 70.53, "latency": 2251.753, "stderr": 3.148, "cost_per_test": 8.575324, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.508, "latency": 2284.395, "stderr": 3.11, "cost_per_test": 11.266649, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 69.978, "latency": 1639.288, "stderr": 2.99, "cost_per_test": 12.066658, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 69.48, "latency": 687.766, "stderr": 2.642, "cost_per_test": 1.223773, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 69.085, "latency": 887.067, "stderr": 2.834, "cost_per_test": 5.886751, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 69.078, "latency": 622.321, "stderr": 2.916, "cost_per_test": 4.126546, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 69.002, "latency": 949.838, "stderr": 2.696, "cost_per_test": 3.785539, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 68.839, "latency": 794.164, "stderr": 2.893, "cost_per_test": 2.110648, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 68.787, "latency": 685.381, "stderr": 2.875, "cost_per_test": 0.241679, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 68.523, "latency": 1011.147, "stderr": 2.534, "cost_per_test": 0.347813, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 68.469, "latency": 3123.343, "stderr": 2.548, "cost_per_test": 3.973753, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 68.39, "latency": 724.799, "stderr": 2.731, "cost_per_test": 6.984969, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 68.222, "latency": 724.523, "stderr": 2.559, "cost_per_test": 5.318355, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 67.821, "latency": 2351.867, "stderr": 2.84, "cost_per_test": 6.249423, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 67.582, "latency": 1843.623, "stderr": 2.457, "cost_per_test": 0.744787, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 67.437, "latency": 2538.573, "stderr": 2.853, "cost_per_test": 11.138836, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 67.327, "latency": 934.097, "stderr": 2.684, "cost_per_test": 1.504318, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 67.048, "latency": 2304.936, "stderr": 2.557, "cost_per_test": 12.38453, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 66.833, "latency": 947.624, "stderr": 2.465, "cost_per_test": 3.412393, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 66.777, "latency": 2506.874, "stderr": 2.483, "cost_per_test": 1.217876, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.2": { "accuracy": 66.693, "latency": 3508.741, "stderr": 2.545, "cost_per_test": 4.028214, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.479, "latency": 769.603, "stderr": 2.444, "cost_per_test": 3.041574, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 66.462, "latency": 2137.932, "stderr": 2.45, "cost_per_test": 3.60704, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 66.431, "latency": 996.439, "stderr": 2.843, "cost_per_test": 0.108806, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 66.358, "latency": 1255.708, "stderr": 3.361, "cost_per_test": 1.290012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 66.148, "latency": 1471.499, "stderr": 2.462, "cost_per_test": 0.10021, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 66.113, "latency": 2260.453, "stderr": 2.524, "cost_per_test": 1.045429, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 66.083, "latency": 714.918, "stderr": 2.627, "cost_per_test": 0.211272, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 65.829, "latency": 1298.369, "stderr": 2.695, "cost_per_test": 2.548879, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 65.691, "latency": 753.531, "stderr": 2.475, "cost_per_test": 3.552764, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 65.628, "latency": 1867.524, "stderr": 2.783, "cost_per_test": 2.161453, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 65.037, "latency": 3133.342, "stderr": 3.212, "cost_per_test": 10.476806, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 64.751, "latency": 3392.83, "stderr": 2.791, "cost_per_test": 7.603845, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 64.715, "latency": 4229.278, "stderr": 2.6, "cost_per_test": 3.867966, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 64.455, "latency": 3591.222, "stderr": 3.144, "cost_per_test": 2.618649, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 64.072, "latency": 1279.259, "stderr": 2.529, "cost_per_test": 4.35085, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 64.0, "latency": 1075.4, "stderr": 2.417, "cost_per_test": 4.744339, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 63.861, "latency": 970.676, "stderr": 2.882, "cost_per_test": 1.749256, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 63.285, "latency": 868.085, "stderr": 2.57, "cost_per_test": 1.015739, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 63.096, "latency": 953.388, "stderr": 3.022, "cost_per_test": 2.127391, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 62.654, "latency": 4418.173, "stderr": 2.53, "cost_per_test": 3.688473, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 62.633, "latency": 5052.751, "stderr": 3.109, "cost_per_test": 0.928309, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 62.215, "latency": 3235.496, "stderr": 3.511, "cost_per_test": 0.287515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 61.447, "latency": 1962.552, "stderr": 2.75, "cost_per_test": 2.74412, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 61.426, "latency": 1537.831, "stderr": 3.037, "cost_per_test": 0.356, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 61.012, "latency": 1976.511, "stderr": 2.558, "cost_per_test": 1.854875, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 59.678, "latency": 2428.134, "stderr": 2.915, "cost_per_test": 2.154897, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 57.916, "latency": 1753.73, "stderr": 2.79, "cost_per_test": 0.210941, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 57.704, "latency": 624.193, "stderr": 2.832, "cost_per_test": 3.148008, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 57.145, "latency": 1845.163, "stderr": 3.085, "cost_per_test": 0.085787, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 56.962, "latency": 2346.663, "stderr": 2.874, "cost_per_test": 0.652699, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 56.692, "latency": 866.492, "stderr": 2.818, "cost_per_test": 0.794261, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 54.67, "latency": 2533.883, "stderr": 3.462, "cost_per_test": 2.010804, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 53.47, "latency": 1134.68, "stderr": 4.974, "cost_per_test": 1.175743, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 49.861, "latency": 2264.595, "stderr": 3.251, "cost_per_test": 0.178991, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 45.101, "latency": 1172.631, "stderr": 3.284, "cost_per_test": 0.220818, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 44.928, "latency": 477.721, "stderr": 2.935, "cost_per_test": 1.149494, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 44.305, "latency": 2788.147, "stderr": 2.869, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 43.805, "latency": 1507.678, "stderr": 3.691, "cost_per_test": 0.484426, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 43.358, "latency": 2060.94, "stderr": 3.098, "cost_per_test": 3.032329, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 43.041, "latency": 1554.841, "stderr": 3.489, "cost_per_test": 1.51523, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 38.155, "latency": 4053.322, "stderr": 3.016, "cost_per_test": 0.869595, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 36.721, "latency": 1129.369, "stderr": 2.448, "cost_per_test": 1.538796, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 30.356, "latency": 302.045, "stderr": 3.656, "cost_per_test": 0.402966, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 29.429, "latency": 281.483, "stderr": 2.696, "cost_per_test": 0.554337, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 28.487, "latency": 703.467, "stderr": 2.773, "cost_per_test": 1.051967, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 27.56, "latency": 752.122, "stderr": 2.762, "cost_per_test": 0.174899, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 17.397, "latency": 2084.305, "stderr": 1.607, "cost_per_test": 0.177946, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 15.038, "latency": 3219.676, "stderr": 2.802, "cost_per_test": 10.895762, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 14.088, "latency": 310.095, "stderr": 2.048, "cost_per_test": 0.429932, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 12.442, "latency": 275.003, "stderr": 1.846, "cost_per_test": 0.4053, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 9.072, "latency": 63.021, "stderr": 1.194, "cost_per_test": 0.064241, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "lbo": { "anthropic/claude-sonnet-5-5": { "accuracy": 69.979, "latency": 2529.291, "stderr": 4.376, "cost_per_test": 11.166488, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 69.64, "latency": 2121.967, "stderr": 4.506, "cost_per_test": 18.889236, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 69.013, "latency": 1926.226, "stderr": 5.999, "cost_per_test": 10.806996, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 68.543, "latency": 3392.293, "stderr": 4.858, "cost_per_test": 21.752123, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.242, "latency": 725.253, "stderr": 4.87, "cost_per_test": 11.104033, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 65.863, "latency": 2092.832, "stderr": 6.469, "cost_per_test": 17.455726, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 64.899, "latency": 4086.921, "stderr": 4.498, "cost_per_test": 5.865028, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 64.429, "latency": 1466.738, "stderr": 5.968, "cost_per_test": 1.183843, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 62.353, "latency": 3664.258, "stderr": 4.122, "cost_per_test": 0.504393, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-astra": { "accuracy": 61.825, "latency": 818.984, "stderr": 6.088, "cost_per_test": 5.720048, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 61.154, "latency": 2156.964, "stderr": 6.027, "cost_per_test": 8.454608, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 60.69, "latency": 1638.99, "stderr": 6.521, "cost_per_test": 9.500848, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 59.967, "latency": 4048.244, "stderr": 6.146, "cost_per_test": 15.354193, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 59.586, "latency": 2261.66, "stderr": 5.995, "cost_per_test": 7.053456, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 59.445, "latency": 694.513, "stderr": 5.635, "cost_per_test": 1.423011, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 59.422, "latency": 1015.586, "stderr": 4.427, "cost_per_test": 10.468715, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 59.307, "latency": 3993.101, "stderr": 5.756, "cost_per_test": 4.011475, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 59.306, "latency": 1052.092, "stderr": 5.554, "cost_per_test": 0.592456, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 59.1, "latency": 1139.036, "stderr": 5.289, "cost_per_test": 0.148123, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 58.933, "latency": 3906.731, "stderr": 5.839, "cost_per_test": 2.216785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 58.56, "latency": 1125.234, "stderr": 6.149, "cost_per_test": 13.510599, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 58.091, "latency": 2919.817, "stderr": 5.407, "cost_per_test": 4.378009, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3": { "accuracy": 58.032, "latency": 3215.995, "stderr": 5.727, "cost_per_test": 6.629973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 57.763, "latency": 1355.484, "stderr": 6.008, "cost_per_test": 0.469175, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 57.501, "latency": 6309.705, "stderr": 5.271, "cost_per_test": 6.771311, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 57.296, "latency": 1425.898, "stderr": 7.242, "cost_per_test": 4.323559, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 56.926, "latency": 2303.334, "stderr": 5.354, "cost_per_test": 0.177028, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 56.876, "latency": 2528.332, "stderr": 6.492, "cost_per_test": 10.613719, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 56.853, "latency": 1491.958, "stderr": 6.098, "cost_per_test": 5.333182, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 56.579, "latency": 2351.579, "stderr": 5.451, "cost_per_test": 6.826336, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 56.449, "latency": 3015.626, "stderr": 6.037, "cost_per_test": 17.932197, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 55.161, "latency": 2900.672, "stderr": 5.482, "cost_per_test": 1.290374, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 54.384, "latency": 3411.868, "stderr": 5.352, "cost_per_test": 17.127734, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 54.183, "latency": 1116.958, "stderr": 5.292, "cost_per_test": 0.338915, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 53.681, "latency": 2499.335, "stderr": 4.947, "cost_per_test": 7.274511, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 53.222, "latency": 1094.17, "stderr": 5.136, "cost_per_test": 3.548798, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 52.536, "latency": 4347.368, "stderr": 5.427, "cost_per_test": 11.216346, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 52.472, "latency": 1173.459, "stderr": 5.339, "cost_per_test": 6.338788, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 51.815, "latency": 4821.481, "stderr": 5.52, "cost_per_test": 6.512343, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 50.698, "latency": 5821.099, "stderr": 5.141, "cost_per_test": 7.431962, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 50.428, "latency": 924.286, "stderr": 6.178, "cost_per_test": 2.789723, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 50.294, "latency": 3745.657, "stderr": 6.108, "cost_per_test": 1.922378, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 49.767, "latency": 1292.905, "stderr": 5.366, "cost_per_test": 1.737996, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 48.684, "latency": 1004.88, "stderr": 6.2, "cost_per_test": 1.925702, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 48.015, "latency": 7977.806, "stderr": 7.203, "cost_per_test": 1.940287, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 47.739, "latency": 2198.075, "stderr": 5.122, "cost_per_test": 0.234191, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 47.18, "latency": 2382.054, "stderr": 4.739, "cost_per_test": 4.717455, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 46.945, "latency": 4382.543, "stderr": 7.351, "cost_per_test": 4.398155, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 46.475, "latency": 2285.283, "stderr": 6.136, "cost_per_test": 0.604921, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 46.47, "latency": 1383.424, "stderr": 5.599, "cost_per_test": 6.810486, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 45.765, "latency": 1213.262, "stderr": 5.589, "cost_per_test": 2.418215, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 45.047, "latency": 2502.244, "stderr": 5.096, "cost_per_test": 0.102455, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 44.93, "latency": 2429.627, "stderr": 5.701, "cost_per_test": 3.032953, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 43.938, "latency": 1042.97, "stderr": 3.924, "cost_per_test": 0.886335, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 43.117, "latency": 3314.566, "stderr": 5.123, "cost_per_test": 3.032183, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 41.608, "latency": 3185.685, "stderr": 5.198, "cost_per_test": 2.327975, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 40.127, "latency": 536.961, "stderr": 4.732, "cost_per_test": 1.241351, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 37.986, "latency": 3113.006, "stderr": 6.186, "cost_per_test": 0.985801, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 37.967, "latency": 784.347, "stderr": 4.006, "cost_per_test": 4.762336, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 35.969, "latency": 2155.325, "stderr": 5.39, "cost_per_test": 0.228694, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 32.592, "latency": 2103.732, "stderr": 4.258, "cost_per_test": 0.681064, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 29.327, "latency": 2282.76, "stderr": 4.694, "cost_per_test": 3.982755, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 28.155, "latency": 911.522, "stderr": 9.61, "cost_per_test": 0.785699, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 27.171, "latency": 5492.16, "stderr": 3.167, "cost_per_test": 0.89295, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 27.071, "latency": 1363.501, "stderr": 5.966, "cost_per_test": 0.244602, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 25.782, "latency": 1815.768, "stderr": 4.076, "cost_per_test": 2.692152, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 23.302, "latency": 3813.919, "stderr": 4.695, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "thinkingmachines/inkling": { "accuracy": 19.094, "latency": 1557.386, "stderr": 4.773, "cost_per_test": 1.564479, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 17.673, "latency": 794.269, "stderr": 4.583, "cost_per_test": 0.167481, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 17.626, "latency": 810.991, "stderr": 4.509, "cost_per_test": 1.090597, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 15.197, "latency": 2472.555, "stderr": 2.454, "cost_per_test": 0.179121, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 11.021, "latency": 342.724, "stderr": 2.191, "cost_per_test": 0.43408, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 9.61, "latency": 3864.563, "stderr": 3.145, "cost_per_test": 13.135423, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.3": { "accuracy": 9.367, "latency": 300.226, "stderr": 3.177, "cost_per_test": 0.595911, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 8.805, "latency": 49.222, "stderr": 2.184, "cost_per_test": 0.037241, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 8.25, "latency": 259.925, "stderr": 2.824, "cost_per_test": 0.375911, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 7.54, "latency": 292.987, "stderr": 3.319, "cost_per_test": 0.338627, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "dcf": { "anthropic/claude-opus-5-5": { "accuracy": 70.139, "latency": 1663.966, "stderr": 7.674, "cost_per_test": 9.328331, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 70.065, "latency": 2289.802, "stderr": 6.7, "cost_per_test": 9.078529, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 70.018, "latency": 1580.513, "stderr": 7.654, "cost_per_test": 5.792757, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 69.88, "latency": 1488.127, "stderr": 6.693, "cost_per_test": 16.129755, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 68.865, "latency": 1571.036, "stderr": 7.542, "cost_per_test": 11.409991, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 66.579, "latency": 1613.553, "stderr": 7.395, "cost_per_test": 3.903815, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 66.334, "latency": 2796.508, "stderr": 7.339, "cost_per_test": 12.058156, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 66.059, "latency": 611.84, "stderr": 6.729, "cost_per_test": 5.594566, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 65.719, "latency": 2275.267, "stderr": 7.019, "cost_per_test": 12.254499, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 65.313, "latency": 784.133, "stderr": 7.529, "cost_per_test": 0.759726, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 64.483, "latency": 649.606, "stderr": 7.442, "cost_per_test": 4.111119, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 64.335, "latency": 3081.904, "stderr": 7.309, "cost_per_test": 3.907974, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 63.655, "latency": 673.252, "stderr": 6.76, "cost_per_test": 6.77049, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 63.142, "latency": 548.146, "stderr": 7.389, "cost_per_test": 0.938299, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 62.771, "latency": 1209.678, "stderr": 7.527, "cost_per_test": 0.105228, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 62.764, "latency": 510.322, "stderr": 7.225, "cost_per_test": 3.850982, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 60.482, "latency": 635.862, "stderr": 7.08, "cost_per_test": 0.221953, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 59.852, "latency": 5051.448, "stderr": 8.339, "cost_per_test": 0.582526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 59.408, "latency": 686.706, "stderr": 6.348, "cost_per_test": 3.066222, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 58.952, "latency": 1395.594, "stderr": 7.294, "cost_per_test": 0.114439, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 58.36, "latency": 975.444, "stderr": 6.889, "cost_per_test": 4.056284, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 58.244, "latency": 2046.477, "stderr": 8.511, "cost_per_test": 5.004626, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 58.121, "latency": 2025.653, "stderr": 7.97, "cost_per_test": 0.306338, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 57.069, "latency": 2515.234, "stderr": 8.273, "cost_per_test": 14.008004, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 56.736, "latency": 703.362, "stderr": 8.236, "cost_per_test": 2.990973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 56.736, "latency": 883.688, "stderr": 7.836, "cost_per_test": 2.839702, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 56.404, "latency": 2886.854, "stderr": 8.129, "cost_per_test": 9.689342, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 55.697, "latency": 1217.179, "stderr": 8.018, "cost_per_test": 2.437825, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 55.562, "latency": 877.044, "stderr": 7.274, "cost_per_test": 1.955636, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 55.108, "latency": 950.59, "stderr": 10.412, "cost_per_test": 0.919685, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 53.395, "latency": 672.589, "stderr": 7.751, "cost_per_test": 1.438464, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 53.131, "latency": 736.548, "stderr": 9.048, "cost_per_test": 1.868027, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 53.0, "latency": 1032.065, "stderr": 9.434, "cost_per_test": 0.350055, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 52.968, "latency": 3370.007, "stderr": 8.572, "cost_per_test": 4.152392, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 52.539, "latency": 2105.827, "stderr": 7.805, "cost_per_test": 0.078854, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 51.288, "latency": 1432.295, "stderr": 7.335, "cost_per_test": 0.38317, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder": { "accuracy": 51.035, "latency": 968.128, "stderr": 8.965, "cost_per_test": 5.11822, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 51.026, "latency": 832.088, "stderr": 8.961, "cost_per_test": 0.237125, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 50.289, "latency": 1789.304, "stderr": 10.058, "cost_per_test": 2.118596, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 50.003, "latency": 1334.104, "stderr": 7.262, "cost_per_test": 3.939808, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 49.837, "latency": 3666.767, "stderr": 9.699, "cost_per_test": 3.166032, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 49.675, "latency": 2553.91, "stderr": 6.323, "cost_per_test": 2.595617, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 49.431, "latency": 1924.758, "stderr": 8.206, "cost_per_test": 0.827029, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 49.134, "latency": 3176.67, "stderr": 8.636, "cost_per_test": 2.313887, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-spark-open": { "accuracy": 47.537, "latency": 5120.532, "stderr": 8.245, "cost_per_test": 4.653132, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 47.387, "latency": 2762.159, "stderr": 9.65, "cost_per_test": 1.211646, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 46.388, "latency": 584.588, "stderr": 7.715, "cost_per_test": 2.495552, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 46.328, "latency": 834.416, "stderr": 8.065, "cost_per_test": 0.996697, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 45.552, "latency": 2214.814, "stderr": 9.279, "cost_per_test": 3.629217, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 44.435, "latency": 816.693, "stderr": 6.242, "cost_per_test": 0.646466, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 43.653, "latency": 2340.579, "stderr": 9.068, "cost_per_test": 0.986264, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 43.63, "latency": 1961.142, "stderr": 7.728, "cost_per_test": 0.219228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 43.53, "latency": 1771.106, "stderr": 7.777, "cost_per_test": 2.896375, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 41.455, "latency": 831.683, "stderr": 8.461, "cost_per_test": 1.193483, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 40.605, "latency": 2573.977, "stderr": 8.924, "cost_per_test": 2.145553, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 38.28, "latency": 3293.001, "stderr": 9.702, "cost_per_test": 7.355698, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 37.537, "latency": 1984.318, "stderr": 7.789, "cost_per_test": 0.64353, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 37.242, "latency": 2177.255, "stderr": 8.715, "cost_per_test": 0.436574, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 36.629, "latency": 1752.148, "stderr": 8.66, "cost_per_test": 2.435709, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 35.881, "latency": 6071.326, "stderr": 9.265, "cost_per_test": 5.023165, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 33.614, "latency": 529.933, "stderr": 7.561, "cost_per_test": 1.27137, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 29.692, "latency": 1745.378, "stderr": 8.845, "cost_per_test": 0.198496, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 29.545, "latency": 2934.041, "stderr": 6.769, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "thinkingmachines/inkling": { "accuracy": 27.54, "latency": 1593.682, "stderr": 7.367, "cost_per_test": 1.523436, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 26.83, "latency": 4575.052, "stderr": 6.424, "cost_per_test": 0.664183, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 24.273, "latency": 1268.624, "stderr": 7.74, "cost_per_test": 0.214907, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 23.323, "latency": 1900.658, "stderr": 8.557, "cost_per_test": 2.673185, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 21.881, "latency": 1115.218, "stderr": 5.961, "cost_per_test": 1.656978, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 21.392, "latency": 294.126, "stderr": 6.312, "cost_per_test": 0.323658, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 19.244, "latency": 741.063, "stderr": 5.315, "cost_per_test": 0.963031, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 18.946, "latency": 796.181, "stderr": 6.493, "cost_per_test": 0.176229, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 11.421, "latency": 277.15, "stderr": 4.91, "cost_per_test": 0.566648, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 11.209, "latency": 2256.093, "stderr": 3.133, "cost_per_test": 0.204987, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 10.067, "latency": 430.644, "stderr": 2.925, "cost_per_test": 0.539724, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 5.098, "latency": 64.7, "stderr": 1.006, "cost_per_test": 0.046823, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.721, "latency": 280.224, "stderr": 1.918, "cost_per_test": 0.411431, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 2.218, "latency": 2845.783, "stderr": 1.847, "cost_per_test": 10.043204, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "ma": { "meta/muse_spark_1_3_max": { "accuracy": 84.188, "latency": 766.584, "stderr": 2.664, "cost_per_test": 2.208341, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 83.906, "latency": 939.372, "stderr": 2.941, "cost_per_test": 0.897061, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 83.873, "latency": 511.166, "stderr": 2.837, "cost_per_test": 3.781208, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 83.364, "latency": 718.865, "stderr": 3.22, "cost_per_test": 3.868043, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 83.066, "latency": 718.565, "stderr": 2.888, "cost_per_test": 5.243954, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 83.021, "latency": 677.534, "stderr": 2.861, "cost_per_test": 1.171055, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 83.002, "latency": 2061.349, "stderr": 3.315, "cost_per_test": 7.900486, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 82.31, "latency": 1803.192, "stderr": 3.275, "cost_per_test": 9.722535, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 81.202, "latency": 1368.605, "stderr": 3.265, "cost_per_test": 14.03427, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 81.03, "latency": 609.151, "stderr": 4.921, "cost_per_test": 5.784968, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 80.97, "latency": 2261.071, "stderr": 4.891, "cost_per_test": 9.012355, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 80.566, "latency": 1534.541, "stderr": 3.449, "cost_per_test": 10.847761, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 80.399, "latency": 1346.235, "stderr": 3.132, "cost_per_test": 4.455983, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 80.091, "latency": 2301.312, "stderr": 3.258, "cost_per_test": 9.449638, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 78.304, "latency": 1362.463, "stderr": 6.661, "cost_per_test": 3.075349, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 77.558, "latency": 762.847, "stderr": 4.857, "cost_per_test": 1.28773, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 77.274, "latency": 2552.008, "stderr": 5.044, "cost_per_test": 2.959945, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-wonder": { "accuracy": 76.852, "latency": 857.035, "stderr": 6.608, "cost_per_test": 4.182315, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 76.265, "latency": 2000.556, "stderr": 6.496, "cost_per_test": 4.338368, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 75.832, "latency": 2458.641, "stderr": 4.698, "cost_per_test": 8.337588, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 75.646, "latency": 668.593, "stderr": 3.903, "cost_per_test": 3.583839, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 74.431, "latency": 2491.543, "stderr": 6.756, "cost_per_test": 0.272898, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 72.73, "latency": 538.544, "stderr": 6.316, "cost_per_test": 2.237213, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 72.051, "latency": 754.204, "stderr": 6.691, "cost_per_test": 0.085662, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 71.668, "latency": 2053.282, "stderr": 5.328, "cost_per_test": 9.204118, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 70.503, "latency": 2932.982, "stderr": 6.534, "cost_per_test": 5.383689, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 69.61, "latency": 524.146, "stderr": 7.279, "cost_per_test": 0.19832, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 68.445, "latency": 1014.816, "stderr": 6.087, "cost_per_test": 4.118564, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 67.965, "latency": 1483.534, "stderr": 8.109, "cost_per_test": 0.557774, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 67.104, "latency": 1958.74, "stderr": 6.819, "cost_per_test": 2.277121, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 66.813, "latency": 866.362, "stderr": 8.09, "cost_per_test": 2.601886, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 66.669, "latency": 1621.3, "stderr": 8.891, "cost_per_test": 1.832399, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 65.272, "latency": 4723.325, "stderr": 8.822, "cost_per_test": 0.51451, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 64.812, "latency": 885.062, "stderr": 6.64, "cost_per_test": 2.686377, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 64.62, "latency": 976.599, "stderr": 9.802, "cost_per_test": 1.016629, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 64.61, "latency": 2412.197, "stderr": 8.543, "cost_per_test": 1.114729, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 64.366, "latency": 1015.132, "stderr": 8.072, "cost_per_test": 2.158016, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 64.205, "latency": 2075.852, "stderr": 5.335, "cost_per_test": 1.570029, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 64.098, "latency": 1173.958, "stderr": 8.179, "cost_per_test": 0.089712, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 64.098, "latency": 1662.395, "stderr": 7.421, "cost_per_test": 0.166174, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.2": { "accuracy": 63.913, "latency": 3655.024, "stderr": 8.842, "cost_per_test": 4.390075, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 63.851, "latency": 848.879, "stderr": 8.634, "cost_per_test": 1.481971, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 63.442, "latency": 3175.157, "stderr": 9.682, "cost_per_test": 3.03352, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 62.29, "latency": 914.971, "stderr": 9.825, "cost_per_test": 0.307102, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 61.504, "latency": 808.487, "stderr": 9.309, "cost_per_test": 0.691326, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 60.705, "latency": 2763.286, "stderr": 8.79, "cost_per_test": 1.95986, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.354, "latency": 1845.107, "stderr": 7.502, "cost_per_test": 0.06733, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 60.144, "latency": 1787.424, "stderr": 9.638, "cost_per_test": 2.589284, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 59.022, "latency": 2155.373, "stderr": 7.824, "cost_per_test": 0.42061, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 58.833, "latency": 646.47, "stderr": 7.829, "cost_per_test": 6.070452, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 57.776, "latency": 661.499, "stderr": 9.403, "cost_per_test": 0.194981, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 57.764, "latency": 1011.493, "stderr": 10.499, "cost_per_test": 0.98052, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 54.761, "latency": 1184.383, "stderr": 9.208, "cost_per_test": 0.283682, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 53.698, "latency": 5014.642, "stderr": 9.124, "cost_per_test": 3.855837, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 53.058, "latency": 1799.44, "stderr": 10.557, "cost_per_test": 0.750715, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 51.9, "latency": 3982.957, "stderr": 9.826, "cost_per_test": 3.038718, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 49.814, "latency": 2013.006, "stderr": 9.217, "cost_per_test": 1.621479, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 48.754, "latency": 1269.268, "stderr": 6.759, "cost_per_test": 1.411152, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.366, "latency": 1834.694, "stderr": 9.642, "cost_per_test": 1.463202, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 48.156, "latency": 764.843, "stderr": 9.249, "cost_per_test": 0.645157, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 44.407, "latency": 1683.669, "stderr": 8.687, "cost_per_test": 0.595991, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 43.422, "latency": 1833.666, "stderr": 8.462, "cost_per_test": 2.337444, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 43.213, "latency": 417.573, "stderr": 7.997, "cost_per_test": 0.952387, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 42.568, "latency": 1703.627, "stderr": 10.048, "cost_per_test": 0.148568, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 37.061, "latency": 2678.053, "stderr": 7.567, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 35.256, "latency": 1046.457, "stderr": 6.11, "cost_per_test": 1.350619, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 34.407, "latency": 321.449, "stderr": 8.107, "cost_per_test": 0.304123, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 30.221, "latency": 1394.035, "stderr": 8.367, "cost_per_test": 0.234461, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 28.544, "latency": 951.39, "stderr": 5.647, "cost_per_test": 0.220899, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 27.454, "latency": 262.581, "stderr": 6.432, "cost_per_test": 0.534988, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 26.299, "latency": 5384.888, "stderr": 6.676, "cost_per_test": 0.928555, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 19.535, "latency": 1814.83, "stderr": 2.913, "cost_per_test": 0.182236, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 15.999, "latency": 715.281, "stderr": 5.791, "cost_per_test": 0.992782, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 10.694, "latency": 2712.804, "stderr": 4.89, "cost_per_test": 8.967043, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 10.003, "latency": 381.481, "stderr": 4.606, "cost_per_test": 0.494944, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 6.751, "latency": 55.632, "stderr": 1.416, "cost_per_test": 0.034741, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 6.35, "latency": 395.701, "stderr": 2.25, "cost_per_test": 0.572506, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "operating": { "anthropic/claude-opus-5-5": { "accuracy": 77.855, "latency": 1547.508, "stderr": 4.588, "cost_per_test": 8.542039, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 75.439, "latency": 590.973, "stderr": 4.244, "cost_per_test": 3.61903, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 74.849, "latency": 1431.686, "stderr": 4.508, "cost_per_test": 13.417399, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 73.92, "latency": 777.595, "stderr": 4.252, "cost_per_test": 0.708017, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 73.545, "latency": 1435.273, "stderr": 6.401, "cost_per_test": 5.596663, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 72.923, "latency": 552.8, "stderr": 5.016, "cost_per_test": 0.926683, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 72.826, "latency": 1414.129, "stderr": 5.142, "cost_per_test": 10.765076, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 72.697, "latency": 578.944, "stderr": 5.168, "cost_per_test": 0.21852, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 71.555, "latency": 2013.788, "stderr": 7.067, "cost_per_test": 7.810405, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 71.088, "latency": 730.612, "stderr": 4.562, "cost_per_test": 0.070005, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 69.46, "latency": 2076.584, "stderr": 4.333, "cost_per_test": 2.881784, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-4-argon": { "accuracy": 69.433, "latency": 2370.247, "stderr": 5.478, "cost_per_test": 10.710486, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 69.063, "latency": 621.677, "stderr": 5.613, "cost_per_test": 6.429094, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 68.56, "latency": 1570.029, "stderr": 6.831, "cost_per_test": 3.836765, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 68.269, "latency": 683.423, "stderr": 4.095, "cost_per_test": 1.509413, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 68.175, "latency": 2879.578, "stderr": 6.228, "cost_per_test": 2.423452, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.257, "latency": 448.591, "stderr": 6.512, "cost_per_test": 4.311989, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 66.699, "latency": 2376.906, "stderr": 6.296, "cost_per_test": 10.392758, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 66.084, "latency": 909.365, "stderr": 6.293, "cost_per_test": 0.323942, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 66.048, "latency": 687.305, "stderr": 6.291, "cost_per_test": 0.206267, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 65.484, "latency": 4297.238, "stderr": 6.526, "cost_per_test": 0.291384, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 65.455, "latency": 2461.863, "stderr": 6.282, "cost_per_test": 7.956599, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 65.354, "latency": 917.117, "stderr": 6.634, "cost_per_test": 4.824542, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.095, "latency": 3452.571, "stderr": 6.674, "cost_per_test": 2.275525, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4-preview": { "accuracy": 64.962, "latency": 1661.63, "stderr": 6.115, "cost_per_test": 0.572349, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-5.5": { "accuracy": 64.478, "latency": 798.917, "stderr": 6.329, "cost_per_test": 2.199544, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 63.995, "latency": 1980.227, "stderr": 6.146, "cost_per_test": 5.428456, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 63.933, "latency": 552.389, "stderr": 6.148, "cost_per_test": 2.501279, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/ember-1": { "accuracy": 63.737, "latency": 1553.905, "stderr": 6.06, "cost_per_test": 1.850664, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 63.149, "latency": 4388.685, "stderr": 6.528, "cost_per_test": 4.195928, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 63.12, "latency": 2211.962, "stderr": 6.007, "cost_per_test": 1.049678, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 62.97, "latency": 712.889, "stderr": 6.584, "cost_per_test": 3.53297, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 62.888, "latency": 3231.995, "stderr": 6.441, "cost_per_test": 3.50696, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 62.726, "latency": 5027.274, "stderr": 6.589, "cost_per_test": 3.735472, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 62.525, "latency": 1022.354, "stderr": 5.851, "cost_per_test": 0.083495, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 62.06, "latency": 3931.146, "stderr": 7.574, "cost_per_test": 0.538579, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 61.935, "latency": 1233.928, "stderr": 7.615, "cost_per_test": 1.114161, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 61.522, "latency": 664.347, "stderr": 7.43, "cost_per_test": 1.786294, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 61.359, "latency": 1821.004, "stderr": 7.414, "cost_per_test": 2.941504, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 61.176, "latency": 2115.382, "stderr": 7.144, "cost_per_test": 10.941835, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 60.803, "latency": 1896.479, "stderr": 7.251, "cost_per_test": 0.845142, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 59.424, "latency": 1023.138, "stderr": 6.665, "cost_per_test": 2.944004, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 58.658, "latency": 904.09, "stderr": 7.143, "cost_per_test": 3.646821, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 58.406, "latency": 462.574, "stderr": 6.897, "cost_per_test": 3.585435, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 58.087, "latency": 506.512, "stderr": 6.208, "cost_per_test": 1.560798, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 58.056, "latency": 2854.463, "stderr": 7.22, "cost_per_test": 5.620452, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 58.018, "latency": 2104.952, "stderr": 7.224, "cost_per_test": 0.479629, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 57.562, "latency": 801.963, "stderr": 7.506, "cost_per_test": 0.788153, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 57.009, "latency": 1034.406, "stderr": 8.278, "cost_per_test": 3.656366, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 56.923, "latency": 2067.12, "stderr": 7.443, "cost_per_test": 1.804151, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 56.409, "latency": 1052.672, "stderr": 9.206, "cost_per_test": 1.128358, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 55.976, "latency": 1316.344, "stderr": 7.061, "cost_per_test": 0.32347, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 55.182, "latency": 1607.152, "stderr": 7.443, "cost_per_test": 0.179455, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 53.807, "latency": 1149.374, "stderr": 8.05, "cost_per_test": 2.721422, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 52.105, "latency": 1827.548, "stderr": 8.48, "cost_per_test": 2.586275, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 51.541, "latency": 1899.325, "stderr": 7.178, "cost_per_test": 0.072876, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 50.834, "latency": 2063.042, "stderr": 8.241, "cost_per_test": 1.973419, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.863, "latency": 1780.736, "stderr": 7.366, "cost_per_test": 1.98786, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 47.176, "latency": 685.554, "stderr": 6.647, "cost_per_test": 0.540271, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 41.599, "latency": 1488.566, "stderr": 6.98, "cost_per_test": 1.850492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 36.655, "latency": 1743.737, "stderr": 7.534, "cost_per_test": 0.543688, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 36.376, "latency": 1243.09, "stderr": 6.107, "cost_per_test": 0.261075, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 34.254, "latency": 2229.57, "stderr": 6.1, "cost_per_test": 0.188595, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 32.737, "latency": 543.184, "stderr": 6.838, "cost_per_test": 1.418567, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 28.566, "latency": 1734.065, "stderr": 8.244, "cost_per_test": 3.12708, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 28.124, "latency": 2684.882, "stderr": 6.907, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 26.679, "latency": 1554.153, "stderr": 6.337, "cost_per_test": 1.98355, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 21.957, "latency": 631.901, "stderr": 5.295, "cost_per_test": 1.00596, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 20.428, "latency": 295.092, "stderr": 7.156, "cost_per_test": 0.367395, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 19.638, "latency": 3382.561, "stderr": 5.356, "cost_per_test": 0.854167, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 17.211, "latency": 703.053, "stderr": 3.686, "cost_per_test": 0.183016, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 16.29, "latency": 265.023, "stderr": 5.497, "cost_per_test": 0.591564, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 10.178, "latency": 3484.28, "stderr": 4.722, "cost_per_test": 12.569555, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 9.843, "latency": 2043.166, "stderr": 2.426, "cost_per_test": 0.201411, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 5.395, "latency": 52.473, "stderr": 1.935, "cost_per_test": 0.034484, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 4.944, "latency": 493.487, "stderr": 2.023, "cost_per_test": 0.303676, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 2.631, "latency": 203.305, "stderr": 1.122, "cost_per_test": 0.366292, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "comps": { "google/gemini-4-argon": { "accuracy": 82.979, "latency": 2301.06, "stderr": 3.492, "cost_per_test": 11.770794, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 82.233, "latency": 2942.999, "stderr": 4.046, "cost_per_test": 24.604393, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 80.89, "latency": 1974.807, "stderr": 3.801, "cost_per_test": 6.137566, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 80.042, "latency": 2463.278, "stderr": 4.256, "cost_per_test": 10.248719, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 79.488, "latency": 996.365, "stderr": 3.841, "cost_per_test": 6.189429, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 77.953, "latency": 1532.002, "stderr": 3.655, "cost_per_test": 12.624116, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 77.925, "latency": 1326.925, "stderr": 4.262, "cost_per_test": 1.766809, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 76.435, "latency": 667.116, "stderr": 4.025, "cost_per_test": 9.096536, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 76.405, "latency": 4242.927, "stderr": 6.136, "cost_per_test": 20.510975, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 75.171, "latency": 1435.222, "stderr": 4.512, "cost_per_test": 7.365041, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 74.609, "latency": 773.405, "stderr": 4.166, "cost_per_test": 7.439465, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 74.413, "latency": 4427.553, "stderr": 4.251, "cost_per_test": 7.609336, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 73.295, "latency": 562.426, "stderr": 6.296, "cost_per_test": 1.730675, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 72.797, "latency": 2442.712, "stderr": 4.198, "cost_per_test": 12.886468, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 71.309, "latency": 814.317, "stderr": 6.203, "cost_per_test": 9.611009, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 70.837, "latency": 667.117, "stderr": 6.266, "cost_per_test": 3.603933, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 70.785, "latency": 1608.159, "stderr": 3.984, "cost_per_test": 16.057068, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 70.474, "latency": 1820.73, "stderr": 6.155, "cost_per_test": 0.373356, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 70.019, "latency": 603.567, "stderr": 3.84, "cost_per_test": 4.730378, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 69.788, "latency": 1282.564, "stderr": 4.035, "cost_per_test": 8.099855, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 69.617, "latency": 1230.32, "stderr": 5.916, "cost_per_test": 0.110268, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 69.384, "latency": 3392.345, "stderr": 6.423, "cost_per_test": 12.922728, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 68.501, "latency": 764.638, "stderr": 6.338, "cost_per_test": 3.28507, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 68.264, "latency": 2820.674, "stderr": 6.462, "cost_per_test": 0.373493, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 65.016, "latency": 2172.982, "stderr": 7.732, "cost_per_test": 7.526137, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 64.798, "latency": 1365.924, "stderr": 5.668, "cost_per_test": 4.913012, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.5": { "accuracy": 64.681, "latency": 925.371, "stderr": 5.533, "cost_per_test": 6.052587, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 64.519, "latency": 2030.658, "stderr": 5.387, "cost_per_test": 14.486164, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 62.157, "latency": 884.572, "stderr": 8.404, "cost_per_test": 3.968341, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.788, "latency": 1969.169, "stderr": 8.547, "cost_per_test": 2.739501, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 61.52, "latency": 3683.558, "stderr": 5.973, "cost_per_test": 8.994714, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 61.395, "latency": 2652.675, "stderr": 4.513, "cost_per_test": 2.521027, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/ember-1": { "accuracy": 60.496, "latency": 1860.074, "stderr": 8.4, "cost_per_test": 3.11394, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.6": { "accuracy": 59.923, "latency": 2889.139, "stderr": 8.325, "cost_per_test": 3.518021, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 59.918, "latency": 1361.083, "stderr": 6.903, "cost_per_test": 0.355729, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 57.495, "latency": 1504.78, "stderr": 8.135, "cost_per_test": 0.909767, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 57.434, "latency": 867.448, "stderr": 6.116, "cost_per_test": 1.285259, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 56.569, "latency": 5366.652, "stderr": 7.833, "cost_per_test": 6.055612, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 54.957, "latency": 1114.726, "stderr": 8.776, "cost_per_test": 2.865188, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.2": { "accuracy": 54.922, "latency": 3252.76, "stderr": 7.703, "cost_per_test": 4.832217, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 54.573, "latency": 621.715, "stderr": 8.831, "cost_per_test": 0.729758, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 54.32, "latency": 4657.725, "stderr": 8.831, "cost_per_test": 3.648233, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 53.293, "latency": 1355.68, "stderr": 7.798, "cost_per_test": 5.470603, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 52.763, "latency": 1421.599, "stderr": 7.575, "cost_per_test": 1.442667, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 52.72, "latency": 948.276, "stderr": 7.879, "cost_per_test": 1.857162, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 50.921, "latency": 2323.758, "stderr": 7.018, "cost_per_test": 3.397738, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 49.774, "latency": 815.93, "stderr": 8.069, "cost_per_test": 0.343239, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 47.341, "latency": 1597.315, "stderr": 7.039, "cost_per_test": 0.399754, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 47.165, "latency": 1464.972, "stderr": 7.148, "cost_per_test": 0.117056, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 46.949, "latency": 1561.313, "stderr": 6.637, "cost_per_test": 0.585614, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 46.417, "latency": 591.245, "stderr": 7.777, "cost_per_test": 1.779222, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 45.324, "latency": 2196.814, "stderr": 9.431, "cost_per_test": 3.744259, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 44.602, "latency": 3203.656, "stderr": 8.667, "cost_per_test": 0.471717, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 44.505, "latency": 768.417, "stderr": 7.615, "cost_per_test": 7.466867, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 43.925, "latency": 630.8, "stderr": 9.09, "cost_per_test": 0.218978, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 42.147, "latency": 1746.604, "stderr": 8.028, "cost_per_test": 1.073792, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 40.991, "latency": 4497.271, "stderr": 8.397, "cost_per_test": 6.33156, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 40.631, "latency": 2163.542, "stderr": 8.571, "cost_per_test": 1.341743, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 40.488, "latency": 3904.48, "stderr": 8.948, "cost_per_test": 3.183265, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 39.489, "latency": 1594.847, "stderr": 9.454, "cost_per_test": 3.025508, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 39.126, "latency": 1981.624, "stderr": 7.846, "cost_per_test": 0.878993, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 33.775, "latency": 2022.524, "stderr": 7.702, "cost_per_test": 0.148543, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling": { "accuracy": 33.031, "latency": 1740.923, "stderr": 7.935, "cost_per_test": 1.989569, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 32.452, "latency": 1598.874, "stderr": 7.631, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 31.299, "latency": 374.146, "stderr": 6.076, "cost_per_test": 0.966148, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 30.763, "latency": 1166.388, "stderr": 8.022, "cost_per_test": 0.360274, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 28.978, "latency": 1720.113, "stderr": 7.167, "cost_per_test": 2.485426, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 27.596, "latency": 2571.313, "stderr": 6.31, "cost_per_test": 0.860893, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 24.596, "latency": 672.622, "stderr": 5.81, "cost_per_test": 1.265802, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 23.584, "latency": 998.214, "stderr": 5.91, "cost_per_test": 1.301526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 23.438, "latency": 709.021, "stderr": 5.22, "cost_per_test": 0.191337, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 18.088, "latency": 3251.299, "stderr": 5.441, "cost_per_test": 10.642102, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.3": { "accuracy": 16.558, "latency": 246.455, "stderr": 4.949, "cost_per_test": 0.816859, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 14.271, "latency": 2902.523, "stderr": 3.351, "cost_per_test": 0.125414, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 11.518, "latency": 317.21, "stderr": 4.015, "cost_per_test": 0.481209, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 10.48, "latency": 101.863, "stderr": 3.475, "cost_per_test": 0.182456, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 9.582, "latency": 323.028, "stderr": 2.547, "cost_per_test": 0.514894, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "dataroom": { "anthropic/claude-opus-5": { "accuracy": 87.681, "latency": 1405.306, "stderr": 3.687, "cost_per_test": 5.420298, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 86.736, "latency": 1370.435, "stderr": 3.653, "cost_per_test": 10.865348, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 86.7, "latency": 2670.97, "stderr": 4.334, "cost_per_test": 3.580316, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 86.481, "latency": 456.096, "stderr": 4.597, "cost_per_test": 3.877832, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 86.27, "latency": 2042.335, "stderr": 3.889, "cost_per_test": 7.856024, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 86.218, "latency": 955.744, "stderr": 3.731, "cost_per_test": 8.512634, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 85.948, "latency": 2182.513, "stderr": 3.719, "cost_per_test": 8.994203, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 85.482, "latency": 1325.144, "stderr": 3.787, "cost_per_test": 1.408937, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 85.362, "latency": 1261.119, "stderr": 4.502, "cost_per_test": 8.349512, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 85.257, "latency": 1361.668, "stderr": 3.973, "cost_per_test": 3.621869, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 85.243, "latency": 539.538, "stderr": 3.94, "cost_per_test": 4.744315, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 85.109, "latency": 1426.808, "stderr": 4.487, "cost_per_test": 7.109521, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 84.909, "latency": 2740.679, "stderr": 4.74, "cost_per_test": 8.190469, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 84.873, "latency": 1493.027, "stderr": 4.792, "cost_per_test": 7.825198, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 84.477, "latency": 757.817, "stderr": 3.887, "cost_per_test": 0.775076, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 84.191, "latency": 1056.83, "stderr": 4.701, "cost_per_test": 2.610351, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 83.92, "latency": 647.302, "stderr": 4.286, "cost_per_test": 1.626948, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 83.58, "latency": 1774.605, "stderr": 4.73, "cost_per_test": 6.064312, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 83.536, "latency": 2691.369, "stderr": 5.308, "cost_per_test": 1.72829, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.2": { "accuracy": 83.444, "latency": 2702.431, "stderr": 3.94, "cost_per_test": 3.41684, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 83.009, "latency": 541.843, "stderr": 5.093, "cost_per_test": 0.282142, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 82.833, "latency": 577.778, "stderr": 5.124, "cost_per_test": 1.561272, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 82.824, "latency": 932.192, "stderr": 6.053, "cost_per_test": 0.295267, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 82.661, "latency": 479.716, "stderr": 5.145, "cost_per_test": 1.919899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 82.468, "latency": 1519.189, "stderr": 5.202, "cost_per_test": 0.083356, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 82.161, "latency": 1061.013, "stderr": 5.744, "cost_per_test": 1.665623, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 81.227, "latency": 818.421, "stderr": 4.694, "cost_per_test": 2.424232, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 81.195, "latency": 1365.657, "stderr": 5.386, "cost_per_test": 1.491495, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 80.921, "latency": 1239.062, "stderr": 5.377, "cost_per_test": 4.451663, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 80.91, "latency": 1823.46, "stderr": 7.193, "cost_per_test": 9.816464, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 80.699, "latency": 1359.288, "stderr": 5.942, "cost_per_test": 1.776778, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 79.71, "latency": 1093.722, "stderr": 5.505, "cost_per_test": 3.695988, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-astra": { "accuracy": 79.319, "latency": 619.459, "stderr": 7.177, "cost_per_test": 4.722862, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 79.232, "latency": 835.941, "stderr": 7.021, "cost_per_test": 5.374192, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 79.17, "latency": 1109.179, "stderr": 4.883, "cost_per_test": 1.64075, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 78.618, "latency": 706.309, "stderr": 5.76, "cost_per_test": 0.749924, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 78.493, "latency": 2415.188, "stderr": 6.337, "cost_per_test": 5.102641, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 78.154, "latency": 2373.78, "stderr": 7.416, "cost_per_test": 2.509083, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 77.965, "latency": 1336.466, "stderr": 6.068, "cost_per_test": 0.144435, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 76.703, "latency": 3356.071, "stderr": 7.551, "cost_per_test": 0.25859, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 75.687, "latency": 2146.883, "stderr": 7.969, "cost_per_test": 1.130429, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 75.536, "latency": 687.984, "stderr": 7.878, "cost_per_test": 0.083815, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 75.069, "latency": 1788.453, "stderr": 6.099, "cost_per_test": 1.852169, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 75.048, "latency": 2882.965, "stderr": 7.703, "cost_per_test": 3.987931, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 74.957, "latency": 1305.618, "stderr": 6.273, "cost_per_test": 0.067242, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 73.863, "latency": 695.253, "stderr": 8.047, "cost_per_test": 0.196161, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 72.965, "latency": 569.256, "stderr": 9.182, "cost_per_test": 2.583986, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 72.803, "latency": 1440.119, "stderr": 7.812, "cost_per_test": 0.298467, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 71.966, "latency": 1605.292, "stderr": 8.9, "cost_per_test": 0.479352, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 71.712, "latency": 1653.009, "stderr": 7.809, "cost_per_test": 0.41702, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-spark-open": { "accuracy": 70.871, "latency": 5754.881, "stderr": 7.808, "cost_per_test": 6.145752, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 68.59, "latency": 963.165, "stderr": 9.526, "cost_per_test": 2.260701, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 67.598, "latency": 1676.234, "stderr": 9.12, "cost_per_test": 3.234823, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 66.878, "latency": 1827.686, "stderr": 9.246, "cost_per_test": 0.962733, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 65.712, "latency": 2590.05, "stderr": 9.142, "cost_per_test": 0.328654, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 64.953, "latency": 1003.567, "stderr": 10.409, "cost_per_test": 1.119264, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 62.572, "latency": 745.646, "stderr": 9.249, "cost_per_test": 0.801873, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 57.687, "latency": 475.52, "stderr": 7.582, "cost_per_test": 1.097176, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 53.816, "latency": 1596.4, "stderr": 10.517, "cost_per_test": 0.62497, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 52.743, "latency": 1882.98, "stderr": 10.633, "cost_per_test": 2.081819, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 51.873, "latency": 1636.223, "stderr": 9.848, "cost_per_test": 1.931057, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 48.286, "latency": 1783.343, "stderr": 9.764, "cost_per_test": 1.595869, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 45.504, "latency": 1989.809, "stderr": 8.843, "cost_per_test": 0.506852, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 43.278, "latency": 1605.689, "stderr": 7.93, "cost_per_test": 2.644441, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 38.189, "latency": 2898.428, "stderr": 10.878, "cost_per_test": 0.185138, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 38.062, "latency": 1261.468, "stderr": 9.384, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 37.31, "latency": 814.361, "stderr": 10.116, "cost_per_test": 0.198897, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 37.032, "latency": 916.512, "stderr": 7.797, "cost_per_test": 1.197173, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 36.634, "latency": 459.043, "stderr": 10.763, "cost_per_test": 0.438128, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 34.521, "latency": 646.185, "stderr": 9.22, "cost_per_test": 0.182941, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 29.632, "latency": 695.125, "stderr": 7.065, "cost_per_test": 0.89217, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 29.176, "latency": 292.666, "stderr": 7.144, "cost_per_test": 0.528428, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 28.602, "latency": 2704.134, "stderr": 6.711, "cost_per_test": 0.188896, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 18.792, "latency": 227.563, "stderr": 7.355, "cost_per_test": 0.408924, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 15.513, "latency": 362.75, "stderr": 5.699, "cost_per_test": 0.379957, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 15.178, "latency": 84.269, "stderr": 5.151, "cost_per_test": 0.050422, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 13.403, "latency": 2169.616, "stderr": 6.618, "cost_per_test": 8.157684, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "misc": { "google/gemini-4-argon": { "accuracy": 73.951, "latency": 1891.012, "stderr": 5.155, "cost_per_test": 8.241005, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 72.15, "latency": 1464.35, "stderr": 3.878, "cost_per_test": 11.677196, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 71.396, "latency": 1677.222, "stderr": 3.041, "cost_per_test": 10.692038, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.972, "latency": 1828.925, "stderr": 3.179, "cost_per_test": 9.159194, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 70.944, "latency": 446.081, "stderr": 4.419, "cost_per_test": 4.177008, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 70.409, "latency": 708.298, "stderr": 4.23, "cost_per_test": 6.778419, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 70.136, "latency": 621.237, "stderr": 3.88, "cost_per_test": 0.241467, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 69.468, "latency": 675.136, "stderr": 4.194, "cost_per_test": 2.493723, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 69.079, "latency": 1941.547, "stderr": 7.409, "cost_per_test": 6.797643, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 68.681, "latency": 545.114, "stderr": 3.132, "cost_per_test": 3.282938, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 68.586, "latency": 834.6, "stderr": 4.353, "cost_per_test": 0.095388, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 66.921, "latency": 1491.379, "stderr": 3.233, "cost_per_test": 4.772687, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 66.77, "latency": 3333.325, "stderr": 3.553, "cost_per_test": 3.659631, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 64.965, "latency": 686.623, "stderr": 3.308, "cost_per_test": 1.053988, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 64.963, "latency": 2124.061, "stderr": 6.679, "cost_per_test": 5.691311, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 64.647, "latency": 985.394, "stderr": 6.895, "cost_per_test": 2.912134, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 64.51, "latency": 819.306, "stderr": 7.134, "cost_per_test": 2.544572, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 64.394, "latency": 583.672, "stderr": 3.006, "cost_per_test": 2.630334, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 64.242, "latency": 1515.655, "stderr": 6.809, "cost_per_test": 3.877366, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 63.627, "latency": 2091.325, "stderr": 6.653, "cost_per_test": 0.093572, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/ember-1": { "accuracy": 63.096, "latency": 1854.798, "stderr": 6.584, "cost_per_test": 2.099692, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 62.99, "latency": 678.229, "stderr": 6.631, "cost_per_test": 2.543941, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 62.192, "latency": 1022.574, "stderr": 6.979, "cost_per_test": 4.497662, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 61.691, "latency": 3079.529, "stderr": 7.18, "cost_per_test": 6.179184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.58, "latency": 903.517, "stderr": 6.195, "cost_per_test": 0.704873, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 60.897, "latency": 1096.615, "stderr": 6.566, "cost_per_test": 0.3924, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 60.775, "latency": 3601.77, "stderr": 6.49, "cost_per_test": 4.199757, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 60.727, "latency": 2820.553, "stderr": 6.652, "cost_per_test": 1.955616, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 60.25, "latency": 3250.123, "stderr": 8.878, "cost_per_test": 2.548576, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 59.814, "latency": 594.866, "stderr": 6.13, "cost_per_test": 3.821522, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 59.549, "latency": 1360.993, "stderr": 6.546, "cost_per_test": 2.506718, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 59.355, "latency": 2930.208, "stderr": 8.87, "cost_per_test": 13.123916, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 59.123, "latency": 1267.483, "stderr": 6.759, "cost_per_test": 4.818541, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 57.746, "latency": 1987.43, "stderr": 6.394, "cost_per_test": 1.909493, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4-preview": { "accuracy": 57.699, "latency": 1706.736, "stderr": 8.855, "cost_per_test": 0.651477, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 57.647, "latency": 811.882, "stderr": 6.554, "cost_per_test": 0.714205, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 57.615, "latency": 2071.462, "stderr": 6.136, "cost_per_test": 0.439265, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder": { "accuracy": 56.936, "latency": 951.464, "stderr": 8.78, "cost_per_test": 3.120758, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 56.397, "latency": 2195.972, "stderr": 8.139, "cost_per_test": 3.379292, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 54.599, "latency": 2093.729, "stderr": 6.131, "cost_per_test": 1.835205, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 54.305, "latency": 2676.363, "stderr": 7.84, "cost_per_test": 12.082044, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 54.045, "latency": 1667.864, "stderr": 6.46, "cost_per_test": 0.069242, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 53.863, "latency": 4327.374, "stderr": 8.275, "cost_per_test": 4.992946, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 53.636, "latency": 833.498, "stderr": 7.974, "cost_per_test": 0.238434, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.787, "latency": 2287.526, "stderr": 7.917, "cost_per_test": 0.920573, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 52.261, "latency": 981.406, "stderr": 11.667, "cost_per_test": 0.966232, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 52.26, "latency": 2692.471, "stderr": 9.595, "cost_per_test": 8.259947, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 50.693, "latency": 1128.107, "stderr": 7.401, "cost_per_test": 2.048008, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 50.644, "latency": 1867.773, "stderr": 6.183, "cost_per_test": 0.178065, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 50.022, "latency": 2177.177, "stderr": 5.091, "cost_per_test": 2.184482, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "stepfun/step-5-preview": { "accuracy": 49.445, "latency": 2643.893, "stderr": 9.336, "cost_per_test": 1.12199, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "grok/grok-4.5": { "accuracy": 49.24, "latency": 1024.593, "stderr": 9.277, "cost_per_test": 1.245392, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 48.822, "latency": 307.154, "stderr": 6.172, "cost_per_test": 0.676879, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 48.727, "latency": 5020.414, "stderr": 6.961, "cost_per_test": 4.421871, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 48.03, "latency": 779.582, "stderr": 9.095, "cost_per_test": 0.767546, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling": { "accuracy": 46.646, "latency": 1849.643, "stderr": 6.78, "cost_per_test": 1.423691, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 46.346, "latency": 2474.0, "stderr": 8.643, "cost_per_test": 2.013106, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 46.012, "latency": 3871.707, "stderr": 8.379, "cost_per_test": 0.552232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 43.378, "latency": 2845.128, "stderr": 8.242, "cost_per_test": 0.147573, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 43.366, "latency": 546.97, "stderr": 8.142, "cost_per_test": 1.744042, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 43.153, "latency": 1656.003, "stderr": 7.697, "cost_per_test": 0.452349, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 42.31, "latency": 2195.876, "stderr": 8.382, "cost_per_test": 0.468188, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 36.32, "latency": 862.741, "stderr": 8.414, "cost_per_test": 0.204405, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 34.713, "latency": 524.864, "stderr": 5.436, "cost_per_test": 0.129778, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 34.646, "latency": 4606.65, "stderr": 10.58, "cost_per_test": 0.246202, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 33.598, "latency": 1205.295, "stderr": 7.38, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 33.174, "latency": 2048.547, "stderr": 7.558, "cost_per_test": 2.626855, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 30.42, "latency": 257.181, "stderr": 7.939, "cost_per_test": 0.378937, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 29.593, "latency": 779.453, "stderr": 7.742, "cost_per_test": 0.955873, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 28.946, "latency": 875.214, "stderr": 5.797, "cost_per_test": 1.250699, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 26.242, "latency": 3004.481, "stderr": 6.484, "cost_per_test": 0.7571, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 20.254, "latency": 2259.011, "stderr": 3.375, "cost_per_test": 0.201566, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.3": { "accuracy": 18.493, "latency": 254.744, "stderr": 5.185, "cost_per_test": 0.815867, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 17.764, "latency": 365.966, "stderr": 4.413, "cost_per_test": 0.655439, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 14.103, "latency": 3007.183, "stderr": 5.948, "cost_per_test": 10.413594, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 12.539, "latency": 304.064, "stderr": 3.581, "cost_per_test": 0.327239, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 8.699, "latency": 59.445, "stderr": 2.021, "cost_per_test": 0.097216, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } } } }