{ "metadata": { "benchmark": "Code Migration v1.1", "slug": "code-migration-v1-1", "description": "Can language models reimplement real-world programs in another language?", "benchmark_id": "code_migration_v1_1", "family": "code_migration", "version": "1.1", "updated": "2026-10-19", "dataset_type": "private", "industry": "coding", "tasks": { "overall": "Overall", "cli": "CLI", "cobol": "COBOL", "code_quality": "Code Quality" }, "models": [ "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.7", "kimi/kimi-k3", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "minimax/MiniMax-M3", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6.1-sol", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 25 }, "tasks": { "overall": { "anthropic/claude-sonnet-5-5": { "accuracy": 86.875, "latency": 14805.742, "stderr": 1.521, "cost_per_test": 43.879213, "token_totals": { "input_tokens": 45761702147, "output_tokens": 167144986, "reasoning_tokens": 79172021, "cache_read_tokens": 45297461218, "cache_write_tokens": 464036227 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 81.012, "latency": 12553.022, "stderr": 3.207, "cost_per_test": 72.39516, "token_totals": { "input_tokens": 39169987377, "output_tokens": 131514806, "reasoning_tokens": 47198663, "cache_read_tokens": 38768480331, "cache_write_tokens": 401329742 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 80.579, "latency": 5221.422, "stderr": 2.336, "cost_per_test": 62.066613, "token_totals": { "input_tokens": 4002222957, "output_tokens": 28255939, "reasoning_tokens": 10478157, "cache_read_tokens": 3922329467, "cache_write_tokens": 79850293 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 77.324, "latency": 5954.402, "stderr": 2.976, "cost_per_test": 9.505623, "token_totals": { "input_tokens": 4796658145, "output_tokens": 31863587, "reasoning_tokens": 12998398, "cache_read_tokens": 4729957671, "cache_write_tokens": 66651049 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 72.355, "latency": 9713.56, "stderr": 3.187, "cost_per_test": 47.931802, "token_totals": { "input_tokens": 28436080448, "output_tokens": 55673508, "reasoning_tokens": 25796839, "cache_read_tokens": 28273954572, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 70.796, "latency": 8528.468, "stderr": 3.723, "cost_per_test": 44.790194, "token_totals": { "input_tokens": 18906563278, "output_tokens": 60680888, "reasoning_tokens": 12677493, "cache_read_tokens": 18376064993, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 67.524, "latency": 8475.469, "stderr": 2.763, "cost_per_test": 1.764297, "token_totals": { "input_tokens": 29616647741, "output_tokens": 57114571, "reasoning_tokens": 26518612, "cache_read_tokens": 29542723456, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 62.79, "latency": 15501.972, "stderr": 3.175, "cost_per_test": 23.584618, "token_totals": { "input_tokens": 11830929459, "output_tokens": 35940066, "reasoning_tokens": 12452527, "cache_read_tokens": 11728671116, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 58.936, "latency": 6814.837, "stderr": 2.656, "cost_per_test": 1.088847, "token_totals": { "input_tokens": 15725809408, "output_tokens": 45507594, "reasoning_tokens": 22859497, "cache_read_tokens": 15695522939, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 55.072, "latency": 10679.835, "stderr": 2.115, "cost_per_test": 34.907391, "token_totals": { "input_tokens": 18444688112, "output_tokens": 47266125, "reasoning_tokens": 23750494, "cache_read_tokens": 18348847770, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 53.217, "latency": 3927.935, "stderr": 2.563, "cost_per_test": 38.051442, "token_totals": { "input_tokens": 6014498708, "output_tokens": 24653408, "reasoning_tokens": 10696135, "cache_read_tokens": 5942478720, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 52.36, "latency": 5158.092, "stderr": 3.489, "cost_per_test": 6.199353, "token_totals": { "input_tokens": 12399726054, "output_tokens": 66282396, "reasoning_tokens": 37421536, "cache_read_tokens": 12266227987, "cache_write_tokens": 133437957 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 51.987, "latency": 10435.782, "stderr": 2.235, "cost_per_test": 12.156614, "token_totals": { "input_tokens": 34168514159, "output_tokens": 64280351, "reasoning_tokens": 27684500, "cache_read_tokens": 34096560384, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 51.647, "latency": 11990.848, "stderr": 2.99, "cost_per_test": 4.480291, "token_totals": { "input_tokens": 42085376103, "output_tokens": 64493842, "reasoning_tokens": 28310169, "cache_read_tokens": 41320368976, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 48.786, "latency": 12508.116, "stderr": 3.876, "cost_per_test": 0.986622, "token_totals": { "input_tokens": 10997149898, "output_tokens": 46213000, "reasoning_tokens": 26396900, "cache_read_tokens": 10859168384, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 47.003, "latency": 11812.433, "stderr": 3.429, "cost_per_test": 8.612923, "token_totals": { "input_tokens": 20951396803, "output_tokens": 48876395, "reasoning_tokens": 26296, "cache_read_tokens": 20839747584, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 46.669, "latency": 3621.471, "stderr": 3.613, "cost_per_test": 0.827763, "token_totals": { "input_tokens": 6056011412, "output_tokens": 29042205, "reasoning_tokens": 16635440, "cache_read_tokens": 6002633060, "cache_write_tokens": 53315421 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 45.364, "latency": 6864.264, "stderr": 3.59, "cost_per_test": 3.57992, "token_totals": { "input_tokens": 7972370226, "output_tokens": 31387457, "reasoning_tokens": 14704493, "cache_read_tokens": 7825408591, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 39.552, "latency": 15010.995, "stderr": 3.537, "cost_per_test": 0.534944, "token_totals": { "input_tokens": 16228858958, "output_tokens": 64563301, "reasoning_tokens": 37165732, "cache_read_tokens": 16107871488, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 38.646, "latency": 9287.211, "stderr": 3.654, "cost_per_test": 15.787246, "token_totals": { "input_tokens": 12489331366, "output_tokens": 28181207, "reasoning_tokens": 13226062, "cache_read_tokens": 12256300882, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 28.587, "latency": 16249.684, "stderr": 3.78, "cost_per_test": 35.748098, "token_totals": { "input_tokens": 14782435512, "output_tokens": 43550358, "reasoning_tokens": 17658063, "cache_read_tokens": 14602391040, "cache_write_tokens": 174460409 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 26.893, "latency": 5021.245, "stderr": 3.229, "cost_per_test": 3.767449, "token_totals": { "input_tokens": 1717177941, "output_tokens": 11784542, "reasoning_tokens": 5997374, "cache_read_tokens": 1500265728, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.898, "latency": 5156.813, "stderr": 3.443, "cost_per_test": 8.391434, "token_totals": { "input_tokens": 8878024286, "output_tokens": 19224096, "reasoning_tokens": 2048097, "cache_read_tokens": 8841406376, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 10.425, "latency": 1076.499, "stderr": 2.324, "cost_per_test": 1.355595, "token_totals": { "input_tokens": 2997168472, "output_tokens": 12937136, "reasoning_tokens": 0, "cache_read_tokens": 2968950528, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 8.821, "latency": 1450.551, "stderr": 2.705, "cost_per_test": 6.133565, "token_totals": { "input_tokens": 2433099263, "output_tokens": 8636715, "reasoning_tokens": 0, "cache_read_tokens": 2410775040, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "cli": { "anthropic/claude-sonnet-5-5": { "accuracy": 82.834, "latency": 16899.509, "stderr": 2.0, "cost_per_test": 52.968857, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 78.35, "latency": 14610.491, "stderr": 2.704, "cost_per_test": 89.979414, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 76.438, "latency": 6281.533, "stderr": 2.064, "cost_per_test": 80.153152, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 74.765, "latency": 7270.974, "stderr": 2.139, "cost_per_test": 12.223041, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 67.839, "latency": 10789.075, "stderr": 3.315, "cost_per_test": 57.991291, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 67.14, "latency": 11847.177, "stderr": 2.69, "cost_per_test": 61.105472, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 59.199, "latency": 10029.729, "stderr": 2.861, "cost_per_test": 2.193325, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 52.721, "latency": 17699.003, "stderr": 3.532, "cost_per_test": 28.113158, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 47.747, "latency": 7584.512, "stderr": 2.674, "cost_per_test": 1.270159, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 44.336, "latency": 4380.258, "stderr": 2.382, "cost_per_test": 1.069181, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 42.147, "latency": 4908.187, "stderr": 2.61, "cost_per_test": 6.315451, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 40.597, "latency": 11308.588, "stderr": 2.798, "cost_per_test": 41.295025, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 40.567, "latency": 4407.713, "stderr": 2.526, "cost_per_test": 47.992099, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 39.197, "latency": 13938.1, "stderr": 2.822, "cost_per_test": 1.109504, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 38.697, "latency": 14287.678, "stderr": 2.993, "cost_per_test": 5.712143, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 37.893, "latency": 7813.984, "stderr": 2.57, "cost_per_test": 4.294849, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 36.815, "latency": 12201.593, "stderr": 2.904, "cost_per_test": 15.150147, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 35.171, "latency": 13290.264, "stderr": 2.503, "cost_per_test": 10.456511, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 27.62, "latency": 11148.606, "stderr": 2.652, "cost_per_test": 19.642889, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 26.977, "latency": 16727.459, "stderr": 2.95, "cost_per_test": 0.652448, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-spark-open": { "accuracy": 15.024, "latency": 5660.496, "stderr": 1.774, "cost_per_test": 4.395183, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 11.449, "latency": 17976.767, "stderr": 2.378, "cost_per_test": 43.026823, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 8.346, "latency": 5429.85, "stderr": 0.976, "cost_per_test": 8.815778, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 3.01, "latency": 1200.466, "stderr": 0.499, "cost_per_test": 1.643369, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 2.613, "latency": 981.835, "stderr": 0.496, "cost_per_test": 2.464356, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "cobol": { "anthropic/claude-sonnet-5-5": { "accuracy": 99.0, "latency": 8524.443, "stderr": 1.0, "cost_per_test": 16.61028, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 98.5, "latency": 8793.577, "stderr": 1.067, "cost_per_test": 15.744488, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 97.5, "latency": 5138.35, "stderr": 2.007, "cost_per_test": 3.176017, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 93.0, "latency": 2041.088, "stderr": 7.0, "cost_per_test": 7.806996, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 93.0, "latency": 8910.877, "stderr": 7.0, "cost_per_test": 9.999, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 92.5, "latency": 3812.688, "stderr": 6.962, "cost_per_test": 0.477214, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 92.5, "latency": 4505.811, "stderr": 6.962, "cost_per_test": 0.54491, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 91.167, "latency": 2488.6, "stderr": 6.903, "cost_per_test": 8.229472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 90.5, "latency": 5100.36, "stderr": 7.904, "cost_per_test": 0.784734, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 89.0, "latency": 6380.613, "stderr": 9.939, "cost_per_test": 19.642399, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 88.0, "latency": 3312.708, "stderr": 9.866, "cost_per_test": 8.410792, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 85.0, "latency": 2004.686, "stderr": 10.028, "cost_per_test": 1.35337, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 83.0, "latency": 5907.808, "stderr": 11.552, "cost_per_test": 5.851057, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 82.5, "latency": 7378.942, "stderr": 11.481, "cost_per_test": 3.082158, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "kimi/kimi-k3": { "accuracy": 80.0, "latency": 11068.432, "stderr": 13.333, "cost_per_test": 13.911925, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 79.667, "latency": 1746.649, "stderr": 11.086, "cost_per_test": 5.186903, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 77.556, "latency": 8218.163, "stderr": 12.987, "cost_per_test": 0.617974, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 77.278, "latency": 9861.601, "stderr": 11.041, "cost_per_test": 0.18243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 71.722, "latency": 3703.024, "stderr": 12.261, "cost_per_test": 4.220319, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 67.778, "latency": 4015.106, "stderr": 12.117, "cost_per_test": 1.435132, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-spark-open": { "accuracy": 62.5, "latency": 3103.489, "stderr": 11.767, "cost_per_test": 1.884248, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 58.556, "latency": 4337.701, "stderr": 13.457, "cost_per_test": 7.118403, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 53.667, "latency": 1345.108, "stderr": 12.563, "cost_per_test": 0.103509, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 32.667, "latency": 704.596, "stderr": 9.173, "cost_per_test": 0.492273, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 27.444, "latency": 2856.7, "stderr": 10.717, "cost_per_test": 17.141191, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "code_quality": { "anthropic/claude-haiku-5-5": { "accuracy": 4.122, "latency": 5158.092, "stderr": 0.027, "cost_per_test": 6.199353, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 3.824, "latency": 14805.742, "stderr": 0.042, "cost_per_test": 43.879213, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 3.816, "latency": 10679.835, "stderr": 0.038, "cost_per_test": 34.907391, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 3.799, "latency": 12553.022, "stderr": 0.044, "cost_per_test": 72.39516, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 3.675, "latency": 6814.837, "stderr": 0.036, "cost_per_test": 1.088847, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 3.611, "latency": 12508.116, "stderr": 0.043, "cost_per_test": 0.986622, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 3.602, "latency": 11812.433, "stderr": 0.039, "cost_per_test": 8.612923, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 3.56, "latency": 15010.995, "stderr": 0.036, "cost_per_test": 0.534944, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 3.552, "latency": 11990.848, "stderr": 0.038, "cost_per_test": 4.480291, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 3.476, "latency": 8475.469, "stderr": 0.033, "cost_per_test": 1.764297, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 3.419, "latency": 5954.402, "stderr": 0.041, "cost_per_test": 9.505623, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 3.408, "latency": 6864.264, "stderr": 0.058, "cost_per_test": 3.57992, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 3.395, "latency": 5221.422, "stderr": 0.032, "cost_per_test": 62.066613, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 3.33, "latency": 16249.684, "stderr": 0.056, "cost_per_test": 35.748098, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 3.261, "latency": 10435.782, "stderr": 0.034, "cost_per_test": 12.156614, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 3.238, "latency": 15501.972, "stderr": 0.043, "cost_per_test": 23.584618, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 3.184, "latency": 3927.935, "stderr": 0.048, "cost_per_test": 38.051442, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 3.107, "latency": 5156.813, "stderr": 0.05, "cost_per_test": 8.391434, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-4-argon": { "accuracy": 2.982, "latency": 8528.468, "stderr": 0.051, "cost_per_test": 44.790194, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 2.975, "latency": 9713.56, "stderr": 0.043, "cost_per_test": 47.931802, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 2.912, "latency": 9287.211, "stderr": 0.049, "cost_per_test": 15.787246, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 2.885, "latency": 3621.471, "stderr": 0.045, "cost_per_test": 0.827763, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 2.398, "latency": 5021.245, "stderr": 0.039, "cost_per_test": 3.767449, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 2.031, "latency": 1076.499, "stderr": 0.032, "cost_per_test": 1.355595, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 1.987, "latency": 1450.551, "stderr": 0.03, "cost_per_test": 6.133565, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } } } }