{ "metadata": { "benchmark": "EMB v1.1", "slug": "emb-v1-1", "description": "Evaluating agents on Excel-based financial modeling tasks, graded by the GPT-6 Luna rubric judge with template scoring fixes", "benchmark_id": "emb_v1_1", "family": "emb", "version": "1.1", "updated": "2026-10-19", "dataset_type": "private", "industry": "finance", "tasks": { "overall": "Overall", "template": "Template", "scratch": "Scratch", "lbo": "LBO", "dcf": "DCF", "ma": "M&A", "operating": "Operating", "comps": "Comps", "dataroom": "Dataroom", "misc": "Misc" }, "models": [ "alibaba/qwen3.8-max", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.7", "kimi/kimi-k3", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "minimax/MiniMax-M3", "minimax/MiniMax-M3.1-flash", "mistralai/mistral-large-4", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6.1-sol", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": true, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 29 }, "tasks": { "overall": { "anthropic/claude-fable-5-1": { "accuracy": 76.728, "latency": 1771.558, "stderr": 2.129, "cost_per_test": 15.93249, "token_totals": { "input_tokens": 454757375, "output_tokens": 12499169, "reasoning_tokens": 8150917, "cache_read_tokens": 378586242, "cache_write_tokens": 76164267 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.899, "latency": 2204.991, "stderr": 2.468, "cost_per_test": 8.792863, "token_totals": { "input_tokens": 2367951142, "output_tokens": 29328538, "reasoning_tokens": 22320203, "cache_read_tokens": 2311427347, "cache_write_tokens": 56507215 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.836, "latency": 2106.434, "stderr": 2.41, "cost_per_test": 11.018974, "token_totals": { "input_tokens": 922243024, "output_tokens": 23704546, "reasoning_tokens": 18569025, "cache_read_tokens": 861165257, "cache_write_tokens": 61067293 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 74.998, "latency": 1684.504, "stderr": 2.141, "cost_per_test": 4.58223, "token_totals": { "input_tokens": 983252171, "output_tokens": 8330149, "reasoning_tokens": 5099887, "cache_read_tokens": 955848482, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 74.828, "latency": 2305.821, "stderr": 2.494, "cost_per_test": 11.8029, "token_totals": { "input_tokens": 1127460088, "output_tokens": 14257556, "reasoning_tokens": 9497334, "cache_read_tokens": 945025185, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 72.871, "latency": 769.197, "stderr": 2.424, "cost_per_test": 8.243862, "token_totals": { "input_tokens": 2960728904, "output_tokens": 21722815, "reasoning_tokens": 13162823, "cache_read_tokens": 2787511478, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 72.683, "latency": 804.28, "stderr": 2.297, "cost_per_test": 5.827063, "token_totals": { "input_tokens": 123989946, "output_tokens": 5542631, "reasoning_tokens": 3051513, "cache_read_tokens": 104449111, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 72.503, "latency": 1034.437, "stderr": 2.414, "cost_per_test": 5.599337, "token_totals": { "input_tokens": 1072958990, "output_tokens": 8128857, "reasoning_tokens": 5046443, "cache_read_tokens": 1021915980, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 71.498, "latency": 2097.023, "stderr": 2.328, "cost_per_test": 6.481284, "token_totals": { "input_tokens": 531961928, "output_tokens": 15529729, "reasoning_tokens": 10549516, "cache_read_tokens": 448911360, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 71.264, "latency": 741.42, "stderr": 2.283, "cost_per_test": 3.074655, "token_totals": { "input_tokens": 410193956, "output_tokens": 9698383, "reasoning_tokens": 7045672, "cache_read_tokens": 393497338, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 71.088, "latency": 1107.811, "stderr": 2.748, "cost_per_test": 1.143199, "token_totals": { "input_tokens": 129631093, "output_tokens": 6056928, "reasoning_tokens": 3495567, "cache_read_tokens": 112922114, "cache_write_tokens": 16703171 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 69.954, "latency": 1041.052, "stderr": 2.542, "cost_per_test": 0.141708, "token_totals": { "input_tokens": 362285754, "output_tokens": 11007243, "reasoning_tokens": 8590626, "cache_read_tokens": 332678280, "cache_write_tokens": 29598418 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 69.552, "latency": 2615.096, "stderr": 2.486, "cost_per_test": 1.329318, "token_totals": { "input_tokens": 1262287639, "output_tokens": 14703425, "reasoning_tokens": 0, "cache_read_tokens": 1227785728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 69.173, "latency": 1971.687, "stderr": 2.395, "cost_per_test": 0.330922, "token_totals": { "input_tokens": 1809205546, "output_tokens": 16117195, "reasoning_tokens": 11510477, "cache_read_tokens": 1797102837, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 69.112, "latency": 2172.231, "stderr": 2.649, "cost_per_test": 3.787827, "token_totals": { "input_tokens": 942177449, "output_tokens": 18177772, "reasoning_tokens": null, "cache_read_tokens": 891631657, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 68.972, "latency": 4904.981, "stderr": 2.339, "cost_per_test": 4.839324, "token_totals": { "input_tokens": 766628987, "output_tokens": 17347923, "reasoning_tokens": null, "cache_read_tokens": 509441536, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 68.946, "latency": 1839.41, "stderr": 2.402, "cost_per_test": 0.787396, "token_totals": { "input_tokens": 824410363, "output_tokens": 10431429, "reasoning_tokens": 7444891, "cache_read_tokens": 799662720, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 68.882, "latency": 1521.906, "stderr": 2.267, "cost_per_test": 0.108634, "token_totals": { "input_tokens": 891374421, "output_tokens": 16056047, "reasoning_tokens": 12655956, "cache_read_tokens": 861570496, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 67.843, "latency": 1847.78, "stderr": 2.29, "cost_per_test": 0.268958, "token_totals": { "input_tokens": 1127000678, "output_tokens": 14186544, "reasoning_tokens": 10606514, "cache_read_tokens": 1112061687, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 67.758, "latency": 780.458, "stderr": 2.565, "cost_per_test": 0.234152, "token_totals": { "input_tokens": 893420650, "output_tokens": 13279281, "reasoning_tokens": 10184478, "cache_read_tokens": 884618493, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 67.347, "latency": 3313.924, "stderr": 2.903, "cost_per_test": 0.328999, "token_totals": { "input_tokens": 847310790, "output_tokens": 17554862, "reasoning_tokens": 14594001, "cache_read_tokens": 813521152, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 66.252, "latency": 4495.202, "stderr": 2.685, "cost_per_test": 0.713033, "token_totals": { "input_tokens": 3932720612, "output_tokens": 29249606, "reasoning_tokens": null, "cache_read_tokens": 3849386240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.986, "latency": 3467.209, "stderr": 2.694, "cost_per_test": 2.655924, "token_totals": { "input_tokens": 599112401, "output_tokens": 12051178, "reasoning_tokens": 8874470, "cache_read_tokens": 572733440, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 65.477, "latency": 1195.527, "stderr": 3.691, "cost_per_test": 1.364559, "token_totals": { "input_tokens": 1482730908, "output_tokens": 25203657, "reasoning_tokens": 19717258, "cache_read_tokens": 1444841999, "cache_write_tokens": 37877007 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 65.128, "latency": 2247.236, "stderr": 2.35, "cost_per_test": 1.08141, "token_totals": { "input_tokens": 864870937, "output_tokens": 15254147, "reasoning_tokens": 11469965, "cache_read_tokens": 855589504, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 61.983, "latency": 5078.614, "stderr": 2.704, "cost_per_test": 5.348821, "token_totals": { "input_tokens": 973042320, "output_tokens": 8495845, "reasoning_tokens": 4888586, "cache_read_tokens": 642573184, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 59.754, "latency": 1298.806, "stderr": 2.537, "cost_per_test": 3.18384, "token_totals": { "input_tokens": 2167916385, "output_tokens": 14444692, "reasoning_tokens": 4485341, "cache_read_tokens": 2146906887, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 40.971, "latency": 1581.835, "stderr": 2.982, "cost_per_test": 1.671783, "token_totals": { "input_tokens": 658436029, "output_tokens": 9281665, "reasoning_tokens": null, "cache_read_tokens": 641195264, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 40.823, "latency": 1173.625, "stderr": 3.21, "cost_per_test": 0.249301, "token_totals": { "input_tokens": 254752442, "output_tokens": 6362193, "reasoning_tokens": null, "cache_read_tokens": 245337472, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "template": { "anthropic/claude-opus-5-5": { "accuracy": 81.385, "latency": 1928.473, "stderr": 2.545, "cost_per_test": 10.771299, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 80.939, "latency": 2158.229, "stderr": 2.616, "cost_per_test": 9.010402, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 80.796, "latency": 1699.436, "stderr": 2.54, "cost_per_test": 16.559081, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 79.861, "latency": 1834.408, "stderr": 2.635, "cost_per_test": 5.314337, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 79.553, "latency": 2452.597, "stderr": 2.887, "cost_per_test": 16.142633, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 76.913, "latency": 813.595, "stderr": 2.827, "cost_per_test": 9.502756, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 76.345, "latency": 1119.036, "stderr": 3.104, "cost_per_test": 7.413135, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 75.648, "latency": 713.238, "stderr": 2.87, "cost_per_test": 3.107736, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 75.081, "latency": 1842.178, "stderr": 2.636, "cost_per_test": 6.713146, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 75.072, "latency": 959.915, "stderr": 3.311, "cost_per_test": 0.996387, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 74.785, "latency": 721.493, "stderr": 3.343, "cost_per_test": 5.767376, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 73.363, "latency": 1180.478, "stderr": 3.965, "cost_per_test": 1.396784, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 72.966, "latency": 3392.353, "stderr": 3.529, "cost_per_test": 0.368856, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 72.745, "latency": 5580.684, "stderr": 2.795, "cost_per_test": 5.810682, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 72.224, "latency": 2723.319, "stderr": 2.986, "cost_per_test": 1.44076, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3": { "accuracy": 71.738, "latency": 2207.215, "stderr": 3.502, "cost_per_test": 3.97223, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 71.726, "latency": 1572.313, "stderr": 3.004, "cost_per_test": 0.117057, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 71.645, "latency": 1872.953, "stderr": 2.773, "cost_per_test": 0.295804, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 71.351, "latency": 1085.666, "stderr": 3.455, "cost_per_test": 0.174611, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 70.829, "latency": 1913.359, "stderr": 2.907, "cost_per_test": 0.346279, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 70.451, "latency": 1835.197, "stderr": 2.801, "cost_per_test": 0.830004, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 70.209, "latency": 845.999, "stderr": 3.111, "cost_per_test": 0.257032, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 70.137, "latency": 3937.654, "stderr": 3.18, "cost_per_test": 0.497756, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.25, "latency": 3340.715, "stderr": 4.166, "cost_per_test": 2.693945, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 64.289, "latency": 2234.018, "stderr": 3.308, "cost_per_test": 1.117392, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 61.805, "latency": 5739.054, "stderr": 3.943, "cost_per_test": 7.009169, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 59.965, "latency": 1237.236, "stderr": 3.325, "cost_per_test": 3.147436, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 38.442, "latency": 1611.127, "stderr": 4.275, "cost_per_test": 1.841659, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 37.824, "latency": 1174.638, "stderr": 4.4, "cost_per_test": 0.278353, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "scratch": { "anthropic/claude-fable-5-1": { "accuracy": 72.674, "latency": 1843.681, "stderr": 2.405, "cost_per_test": 15.305899, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 70.865, "latency": 887.067, "stderr": 2.54, "cost_per_test": 5.886751, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 70.815, "latency": 2251.753, "stderr": 3.153, "cost_per_test": 8.575324, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.249, "latency": 2284.395, "stderr": 3.169, "cost_per_test": 11.266649, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 70.032, "latency": 1534.599, "stderr": 2.614, "cost_per_test": 3.850123, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 69.999, "latency": 2159.046, "stderr": 2.693, "cost_per_test": 7.463166, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 68.848, "latency": 949.838, "stderr": 2.723, "cost_per_test": 3.785539, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 68.544, "latency": 724.799, "stderr": 2.792, "cost_per_test": 6.984969, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 68.125, "latency": 996.439, "stderr": 2.488, "cost_per_test": 0.108806, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 67.548, "latency": 2351.867, "stderr": 2.89, "cost_per_test": 6.249423, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 67.325, "latency": 2030.015, "stderr": 2.64, "cost_per_test": 0.315566, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 67.115, "latency": 1843.623, "stderr": 2.466, "cost_per_test": 0.744787, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 66.937, "latency": 1255.708, "stderr": 3.427, "cost_per_test": 1.290012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.759, "latency": 769.603, "stderr": 2.493, "cost_per_test": 3.041574, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 66.568, "latency": 2506.874, "stderr": 2.529, "cost_per_test": 1.217876, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3": { "accuracy": 66.467, "latency": 2137.932, "stderr": 2.55, "cost_per_test": 3.60704, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 66.269, "latency": 3591.222, "stderr": 2.52, "cost_per_test": 2.618649, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 65.681, "latency": 1471.499, "stderr": 2.482, "cost_per_test": 0.10021, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 65.433, "latency": 2260.453, "stderr": 2.548, "cost_per_test": 1.045429, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 65.238, "latency": 714.918, "stderr": 2.741, "cost_per_test": 0.211272, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 65.088, "latency": 4229.278, "stderr": 2.546, "cost_per_test": 3.867966, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 63.888, "latency": 1822.607, "stderr": 2.408, "cost_per_test": 0.242112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 62.815, "latency": 3235.496, "stderr": 3.554, "cost_per_test": 0.287515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 62.313, "latency": 5052.751, "stderr": 3.049, "cost_per_test": 0.928309, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 61.72, "latency": 4418.173, "stderr": 2.699, "cost_per_test": 3.688473, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 59.207, "latency": 1360.375, "stderr": 2.794, "cost_per_test": 3.220244, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 57.654, "latency": 1210.577, "stderr": 4.548, "cost_per_test": 1.331689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 43.535, "latency": 1172.631, "stderr": 3.174, "cost_per_test": 0.220818, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 42.458, "latency": 1554.841, "stderr": 3.442, "cost_per_test": 1.51523, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "lbo": { "anthropic/claude-sonnet-5-5": { "accuracy": 68.661, "latency": 2529.291, "stderr": 4.232, "cost_per_test": 11.166488, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 68.328, "latency": 1926.226, "stderr": 5.894, "cost_per_test": 10.806996, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 67.988, "latency": 2121.967, "stderr": 4.67, "cost_per_test": 18.889236, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 67.162, "latency": 3392.293, "stderr": 4.775, "cost_per_test": 21.752123, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 64.223, "latency": 1638.99, "stderr": 5.103, "cost_per_test": 9.500848, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 63.763, "latency": 1466.738, "stderr": 5.895, "cost_per_test": 1.183843, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 63.496, "latency": 2261.66, "stderr": 4.727, "cost_per_test": 7.053456, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 62.463, "latency": 4170.026, "stderr": 4.415, "cost_per_test": 0.419172, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 62.17, "latency": 1125.234, "stderr": 4.799, "cost_per_test": 13.510599, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 61.557, "latency": 3664.258, "stderr": 4.172, "cost_per_test": 0.504393, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 61.413, "latency": 3215.995, "stderr": 4.383, "cost_per_test": 6.629973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 61.222, "latency": 3906.731, "stderr": 4.338, "cost_per_test": 2.216785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-astra": { "accuracy": 61.219, "latency": 818.984, "stderr": 6.096, "cost_per_test": 5.720048, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 60.685, "latency": 1491.958, "stderr": 4.786, "cost_per_test": 5.333182, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 60.621, "latency": 6309.705, "stderr": 3.748, "cost_per_test": 6.771311, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 60.603, "latency": 2528.332, "stderr": 5.154, "cost_per_test": 10.613719, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 59.512, "latency": 2303.334, "stderr": 3.727, "cost_per_test": 0.177028, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 58.626, "latency": 3945.157, "stderr": 4.241, "cost_per_test": 0.546001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 57.937, "latency": 1139.036, "stderr": 5.108, "cost_per_test": 0.148123, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 57.8, "latency": 2900.672, "stderr": 3.828, "cost_per_test": 1.290374, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 56.38, "latency": 1116.958, "stderr": 3.914, "cost_per_test": 0.338915, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.76, "latency": 3745.657, "stderr": 5.047, "cost_per_test": 1.922378, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 50.357, "latency": 4382.543, "stderr": 6.523, "cost_per_test": 4.398155, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 50.003, "latency": 7977.806, "stderr": 6.387, "cost_per_test": 1.940287, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 49.132, "latency": 1186.827, "stderr": 9.09, "cost_per_test": 1.420952, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 48.325, "latency": 5821.099, "stderr": 5.029, "cost_per_test": 7.431962, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 41.557, "latency": 2020.752, "stderr": 3.959, "cost_per_test": 5.339965, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 26.761, "latency": 1363.501, "stderr": 5.885, "cost_per_test": 0.244602, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 20.262, "latency": 1557.386, "stderr": 4.457, "cost_per_test": 1.564479, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "dcf": { "anthropic/claude-sonnet-5-5": { "accuracy": 72.971, "latency": 2289.802, "stderr": 6.726, "cost_per_test": 9.078529, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 71.518, "latency": 1488.127, "stderr": 6.765, "cost_per_test": 16.129755, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 69.87, "latency": 1663.966, "stderr": 7.694, "cost_per_test": 9.328331, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 67.129, "latency": 784.133, "stderr": 7.86, "cost_per_test": 0.759726, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 66.115, "latency": 673.252, "stderr": 6.886, "cost_per_test": 6.77049, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 66.027, "latency": 649.606, "stderr": 7.642, "cost_per_test": 4.111119, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 65.831, "latency": 2762.159, "stderr": 6.896, "cost_per_test": 1.211646, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "meta/muse-wonder-full": { "accuracy": 65.743, "latency": 1613.553, "stderr": 7.305, "cost_per_test": 3.903815, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 65.549, "latency": 2275.267, "stderr": 7.008, "cost_per_test": 12.254499, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 65.309, "latency": 5051.448, "stderr": 6.605, "cost_per_test": 0.582526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 65.083, "latency": 2046.477, "stderr": 7.5, "cost_per_test": 5.004626, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 64.83, "latency": 1209.678, "stderr": 7.563, "cost_per_test": 0.105228, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 64.792, "latency": 2025.653, "stderr": 6.607, "cost_per_test": 0.306338, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 64.432, "latency": 703.362, "stderr": 7.236, "cost_per_test": 2.990973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 63.677, "latency": 2214.814, "stderr": 7.269, "cost_per_test": 3.629217, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-wonder": { "accuracy": 63.666, "latency": 968.128, "stderr": 7.151, "cost_per_test": 5.11822, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 63.077, "latency": 1213.846, "stderr": 8.618, "cost_per_test": 1.408154, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 61.621, "latency": 1395.594, "stderr": 7.18, "cost_per_test": 0.114439, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 60.904, "latency": 6071.326, "stderr": 7.34, "cost_per_test": 5.023165, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 59.952, "latency": 1924.758, "stderr": 6.175, "cost_per_test": 0.827029, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 59.853, "latency": 832.088, "stderr": 7.038, "cost_per_test": 0.237125, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 59.71, "latency": 3628.584, "stderr": 7.04, "cost_per_test": 0.336662, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 58.739, "latency": 2759.397, "stderr": 6.7, "cost_per_test": 0.244351, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 58.083, "latency": 2340.579, "stderr": 6.333, "cost_per_test": 0.986264, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 54.207, "latency": 3176.67, "stderr": 7.732, "cost_per_test": 2.313887, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 52.925, "latency": 1306.134, "stderr": 6.285, "cost_per_test": 3.115677, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "meta/muse-spark-open": { "accuracy": 49.913, "latency": 5120.532, "stderr": 7.255, "cost_per_test": 4.653132, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 34.665, "latency": 1268.624, "stderr": 6.949, "cost_per_test": 0.214907, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 27.422, "latency": 1593.682, "stderr": 7.154, "cost_per_test": 1.523436, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "ma": { "meta/muse-wonder-full": { "accuracy": 83.423, "latency": 1362.463, "stderr": 3.078, "cost_per_test": 3.075349, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 83.08, "latency": 939.372, "stderr": 3.013, "cost_per_test": 0.897061, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 82.904, "latency": 718.565, "stderr": 2.964, "cost_per_test": 5.243954, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 82.294, "latency": 2061.349, "stderr": 3.526, "cost_per_test": 7.900486, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 82.054, "latency": 1803.192, "stderr": 3.375, "cost_per_test": 9.722535, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 80.796, "latency": 2491.543, "stderr": 4.157, "cost_per_test": 0.272898, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 80.736, "latency": 2000.556, "stderr": 3.064, "cost_per_test": 4.338368, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 80.537, "latency": 1368.605, "stderr": 3.444, "cost_per_test": 14.03427, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 80.373, "latency": 857.035, "stderr": 3.445, "cost_per_test": 4.182315, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 80.211, "latency": 2261.071, "stderr": 5.082, "cost_per_test": 9.012355, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 79.906, "latency": 609.151, "stderr": 5.165, "cost_per_test": 5.784968, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 79.218, "latency": 1787.424, "stderr": 4.719, "cost_per_test": 2.589284, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 76.808, "latency": 1483.534, "stderr": 3.532, "cost_per_test": 0.557774, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k3": { "accuracy": 76.64, "latency": 538.544, "stderr": 3.46, "cost_per_test": 2.237213, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 75.714, "latency": 661.499, "stderr": 4.754, "cost_per_test": 0.194981, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 74.85, "latency": 754.204, "stderr": 4.402, "cost_per_test": 0.085662, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 74.652, "latency": 2412.197, "stderr": 5.295, "cost_per_test": 1.114729, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 74.594, "latency": 1173.958, "stderr": 3.901, "cost_per_test": 0.089712, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 74.391, "latency": 2537.751, "stderr": 3.297, "cost_per_test": 0.195966, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 74.061, "latency": 3982.957, "stderr": 3.92, "cost_per_test": 3.038718, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 73.562, "latency": 4723.325, "stderr": 4.939, "cost_per_test": 0.51451, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 73.081, "latency": 2347.799, "stderr": 5.038, "cost_per_test": 0.286688, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 73.009, "latency": 2763.286, "stderr": 6.026, "cost_per_test": 1.95986, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 72.565, "latency": 1799.44, "stderr": 5.27, "cost_per_test": 0.750715, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 71.133, "latency": 5014.642, "stderr": 5.319, "cost_per_test": 3.855837, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 70.608, "latency": 1122.869, "stderr": 5.78, "cost_per_test": 1.905749, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 62.94, "latency": 1061.686, "stderr": 9.591, "cost_per_test": 0.991166, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 52.904, "latency": 1269.268, "stderr": 5.857, "cost_per_test": 1.411152, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 42.095, "latency": 1394.035, "stderr": 6.919, "cost_per_test": 0.234461, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "operating": { "anthropic/claude-opus-5-5": { "accuracy": 77.742, "latency": 1547.508, "stderr": 4.627, "cost_per_test": 8.542039, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 75.505, "latency": 590.973, "stderr": 4.345, "cost_per_test": 3.61903, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 75.175, "latency": 1431.686, "stderr": 4.514, "cost_per_test": 13.417399, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 74.631, "latency": 1570.029, "stderr": 5.192, "cost_per_test": 3.836765, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 74.424, "latency": 777.595, "stderr": 4.286, "cost_per_test": 0.708017, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 71.911, "latency": 687.305, "stderr": 4.859, "cost_per_test": 0.206267, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 71.557, "latency": 4297.238, "stderr": 5.133, "cost_per_test": 0.291384, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 71.435, "latency": 2013.788, "stderr": 7.166, "cost_per_test": 7.810405, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 70.796, "latency": 730.612, "stderr": 4.562, "cost_per_test": 0.070005, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 70.638, "latency": 917.117, "stderr": 5.175, "cost_per_test": 4.824542, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "tencent/hy4-preview": { "accuracy": 70.209, "latency": 1661.63, "stderr": 4.46, "cost_per_test": 0.572349, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 70.097, "latency": 3931.146, "stderr": 4.736, "cost_per_test": 0.538579, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 69.926, "latency": 552.389, "stderr": 4.75, "cost_per_test": 2.501279, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 69.918, "latency": 2370.247, "stderr": 5.486, "cost_per_test": 10.710486, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 69.488, "latency": 1821.004, "stderr": 5.147, "cost_per_test": 2.941504, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 69.366, "latency": 3452.571, "stderr": 5.108, "cost_per_test": 2.275525, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 69.19, "latency": 621.677, "stderr": 5.563, "cost_per_test": 6.429094, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "stepfun/step-5-preview": { "accuracy": 68.853, "latency": 2211.962, "stderr": 4.445, "cost_per_test": 1.049678, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 68.821, "latency": 5027.274, "stderr": 5.339, "cost_per_test": 3.735472, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 68.798, "latency": 1980.227, "stderr": 4.574, "cost_per_test": 5.428456, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 68.645, "latency": 4388.685, "stderr": 5.159, "cost_per_test": 4.195928, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 68.181, "latency": 1896.479, "stderr": 4.361, "cost_per_test": 0.845142, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 67.22, "latency": 1022.354, "stderr": 3.96, "cost_per_test": 0.083495, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 66.456, "latency": 861.196, "stderr": 4.786, "cost_per_test": 0.189071, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 65.965, "latency": 842.327, "stderr": 4.694, "cost_per_test": 0.28596, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 62.734, "latency": 1172.526, "stderr": 7.533, "cost_per_test": 1.37752, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 62.347, "latency": 1106.775, "stderr": 5.35, "cost_per_test": 1.873974, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 43.846, "latency": 1488.566, "stderr": 6.396, "cost_per_test": 1.850492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 40.964, "latency": 1243.09, "stderr": 5.899, "cost_per_test": 0.261075, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "comps": { "google/gemini-4-argon": { "accuracy": 82.94, "latency": 2301.06, "stderr": 3.441, "cost_per_test": 11.770794, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 81.998, "latency": 2942.999, "stderr": 4.075, "cost_per_test": 24.604393, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 80.197, "latency": 1974.807, "stderr": 3.914, "cost_per_test": 6.137566, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 79.406, "latency": 2463.278, "stderr": 4.218, "cost_per_test": 10.248719, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 77.902, "latency": 996.365, "stderr": 4.38, "cost_per_test": 6.189429, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 77.701, "latency": 1326.925, "stderr": 4.238, "cost_per_test": 1.766809, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 76.524, "latency": 1532.002, "stderr": 3.865, "cost_per_test": 12.624116, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 76.386, "latency": 667.117, "stderr": 4.323, "cost_per_test": 3.603933, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 76.029, "latency": 667.116, "stderr": 4.257, "cost_per_test": 9.096536, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.658, "latency": 4242.927, "stderr": 6.46, "cost_per_test": 20.510975, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 72.44, "latency": 2172.982, "stderr": 4.643, "cost_per_test": 7.526137, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 71.581, "latency": 770.52, "stderr": 3.537, "cost_per_test": 0.255073, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 70.771, "latency": 790.726, "stderr": 4.335, "cost_per_test": 0.331686, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 70.47, "latency": 5366.652, "stderr": 3.877, "cost_per_test": 6.055612, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 69.668, "latency": 1820.73, "stderr": 6.122, "cost_per_test": 0.373356, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 69.274, "latency": 1230.32, "stderr": 5.975, "cost_per_test": 0.110268, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 68.545, "latency": 1504.78, "stderr": 3.972, "cost_per_test": 0.909767, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 67.988, "latency": 630.8, "stderr": 5.6, "cost_per_test": 0.218978, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 67.982, "latency": 2820.674, "stderr": 6.386, "cost_per_test": 0.373493, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 67.744, "latency": 3203.656, "stderr": 4.242, "cost_per_test": 0.471717, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3": { "accuracy": 67.603, "latency": 2196.814, "stderr": 5.62, "cost_per_test": 3.744259, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 66.843, "latency": 2163.542, "stderr": 4.301, "cost_per_test": 1.341743, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.706, "latency": 4657.725, "stderr": 5.915, "cost_per_test": 3.648233, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.233, "latency": 1969.169, "stderr": 8.554, "cost_per_test": 2.739501, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 60.114, "latency": 946.911, "stderr": 4.388, "cost_per_test": 4.769582, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 59.312, "latency": 1746.604, "stderr": 5.247, "cost_per_test": 1.073792, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 56.044, "latency": 4497.271, "stderr": 6.916, "cost_per_test": 6.33156, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 45.63, "latency": 1166.388, "stderr": 6.065, "cost_per_test": 0.360274, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 40.703, "latency": 1740.923, "stderr": 7.019, "cost_per_test": 1.989569, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "dataroom": { "anthropic/claude-fable-5-1": { "accuracy": 88.518, "latency": 1370.435, "stderr": 3.161, "cost_per_test": 10.865348, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 87.797, "latency": 955.744, "stderr": 3.459, "cost_per_test": 8.512634, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 87.732, "latency": 835.941, "stderr": 3.186, "cost_per_test": 5.374192, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 87.706, "latency": 2042.335, "stderr": 3.554, "cost_per_test": 7.856024, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 86.553, "latency": 1361.668, "stderr": 3.59, "cost_per_test": 3.621869, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 86.498, "latency": 757.817, "stderr": 3.402, "cost_per_test": 0.775076, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 86.495, "latency": 919.03, "stderr": 4.278, "cost_per_test": 0.315607, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 86.197, "latency": 1493.027, "stderr": 4.622, "cost_per_test": 7.825198, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 86.056, "latency": 3356.071, "stderr": 4.412, "cost_per_test": 0.25859, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-astra": { "accuracy": 85.964, "latency": 619.459, "stderr": 3.525, "cost_per_test": 4.722862, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 85.779, "latency": 1325.144, "stderr": 3.659, "cost_per_test": 1.408937, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 84.974, "latency": 1426.808, "stderr": 4.561, "cost_per_test": 7.109521, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 84.614, "latency": 1774.605, "stderr": 4.739, "cost_per_test": 6.064312, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 84.279, "latency": 2691.369, "stderr": 5.196, "cost_per_test": 1.72829, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "stepfun/step-5-preview": { "accuracy": 83.686, "latency": 2146.883, "stderr": 5.801, "cost_per_test": 1.130429, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "kimi/kimi-k3": { "accuracy": 83.227, "latency": 479.716, "stderr": 5.109, "cost_per_test": 1.919899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 83.135, "latency": 2882.965, "stderr": 4.752, "cost_per_test": 3.987931, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 82.616, "latency": 1519.189, "stderr": 5.134, "cost_per_test": 0.083356, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 82.341, "latency": 1596.4, "stderr": 5.553, "cost_per_test": 0.62497, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-luna": { "accuracy": 82.064, "latency": 687.984, "stderr": 5.286, "cost_per_test": 0.083815, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 81.624, "latency": 1676.234, "stderr": 5.41, "cost_per_test": 3.234823, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 81.398, "latency": 1827.686, "stderr": 5.376, "cost_per_test": 0.962733, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 80.321, "latency": 2590.05, "stderr": 5.598, "cost_per_test": 0.328654, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 80.223, "latency": 826.821, "stderr": 5.877, "cost_per_test": 0.175491, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 79.935, "latency": 695.253, "stderr": 5.964, "cost_per_test": 0.196161, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 79.535, "latency": 5754.881, "stderr": 5.952, "cost_per_test": 6.145752, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 77.645, "latency": 1231.363, "stderr": 5.761, "cost_per_test": 2.2131, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 56.62, "latency": 814.361, "stderr": 9.884, "cost_per_test": 0.198897, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 52.197, "latency": 1636.223, "stderr": 9.707, "cost_per_test": 1.931057, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "misc": { "google/gemini-4-argon": { "accuracy": 73.044, "latency": 1891.012, "stderr": 5.253, "cost_per_test": 8.241005, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 71.363, "latency": 1464.35, "stderr": 3.781, "cost_per_test": 11.677196, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 71.0, "latency": 1828.925, "stderr": 3.12, "cost_per_test": 9.159194, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 70.941, "latency": 1515.655, "stderr": 3.367, "cost_per_test": 3.877366, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-luna": { "accuracy": 69.535, "latency": 834.6, "stderr": 4.03, "cost_per_test": 0.095388, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 68.887, "latency": 708.298, "stderr": 4.114, "cost_per_test": 6.778419, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 68.819, "latency": 1941.547, "stderr": 7.35, "cost_per_test": 6.797643, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 68.212, "latency": 2124.061, "stderr": 3.393, "cost_per_test": 5.691311, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 67.552, "latency": 678.229, "stderr": 3.42, "cost_per_test": 2.543941, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 67.336, "latency": 2091.325, "stderr": 3.62, "cost_per_test": 0.093572, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 66.966, "latency": 1706.736, "stderr": 4.194, "cost_per_test": 0.651477, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "stepfun/step-5-preview": { "accuracy": 65.775, "latency": 2643.893, "stderr": 3.402, "cost_per_test": 1.12199, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 65.723, "latency": 977.252, "stderr": 3.32, "cost_per_test": 0.334964, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 64.973, "latency": 2820.553, "stderr": 3.686, "cost_per_test": 1.955616, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 64.885, "latency": 1125.671, "stderr": 2.834, "cost_per_test": 0.247506, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 64.79, "latency": 4327.374, "stderr": 3.863, "cost_per_test": 4.992946, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 63.597, "latency": 2287.526, "stderr": 3.081, "cost_per_test": 0.920573, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder": { "accuracy": 62.987, "latency": 951.464, "stderr": 6.965, "cost_per_test": 3.120758, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 62.522, "latency": 833.498, "stderr": 3.357, "cost_per_test": 0.238434, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.486, "latency": 903.517, "stderr": 6.168, "cost_per_test": 0.704873, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 60.76, "latency": 2195.972, "stderr": 6.249, "cost_per_test": 3.379292, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 60.636, "latency": 594.866, "stderr": 6.196, "cost_per_test": 3.821522, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 60.284, "latency": 5020.414, "stderr": 3.136, "cost_per_test": 4.421871, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 56.973, "latency": 1046.155, "stderr": 10.686, "cost_per_test": 1.073444, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 56.729, "latency": 3871.707, "stderr": 5.749, "cost_per_test": 0.552232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 53.083, "latency": 1336.836, "stderr": 2.512, "cost_per_test": 2.644328, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 49.464, "latency": 1849.643, "stderr": 5.472, "cost_per_test": 1.423691, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 39.029, "latency": 862.741, "stderr": 7.528, "cost_per_test": 0.204405, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 38.688, "latency": 4606.65, "stderr": 10.109, "cost_per_test": 0.246202, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null } } } }