{ "metadata": { "benchmark": "Vibe Code Bench v1.1", "slug": "vibe-code", "description": "Can models build web applications from scratch?", "benchmark_id": "vibe_code_bench", "family": "vibe_code_bench", "version": "1.1", "updated": "2026-10-02", "dataset_type": "private", "industry": "coding", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3-max", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-4-8-claude-code", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-4-6-claude-code", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "cursor/composer-2.5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "devin/swe-1-6-fast", "fireworks/deepseek-v3p2-thinking", "fireworks/ember-1", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "grok/grok-build-0.1", "inception/mercury-2.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k2.7-code", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2603", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.2-codex", "openai/gpt-5.3-codex", "openai/gpt-5.4", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.5-codex", "openai/gpt-5.5-factory", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "poolside/laguna-m.1", "poolside/laguna-xs.2", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-4.6", "zai/glm-5-thinking", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 108 }, "tasks": { "overall": { "anthropic/claude-sonnet-5-5": { "accuracy": 92.392, "latency": 3672.106, "stderr": 1.26, "cost_per_test": 31.250578, "token_totals": { "input_tokens": 5822927217, "output_tokens": 25491041, "reasoning_tokens": 11893204, "cache_read_tokens": 5760733058, "cache_write_tokens": 62167153 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "google/gemini-4-argon": { "accuracy": 91.906, "latency": 2247.675, "stderr": 1.899, "cost_per_test": 8.10797, "token_totals": { "input_tokens": 451864686, "output_tokens": 5436370, "reasoning_tokens": 849287, "cache_read_tokens": 397575696, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "anthropic/claude-fable-5": { "accuracy": 90.352, "latency": 3711.026, "stderr": 2.097, "cost_per_test": 41.714694, "token_totals": { "input_tokens": 1628443000, "output_tokens": 8412174, "reasoning_tokens": 2231064, "cache_read_tokens": 1604184760, "cache_write_tokens": 24245069 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-opus-5-5": { "accuracy": 90.294, "latency": 5766.524, "stderr": 1.526, "cost_per_test": 57.922811, "token_totals": { "input_tokens": 5596583425, "output_tokens": 19722501, "reasoning_tokens": 7457724, "cache_read_tokens": 5539450465, "cache_write_tokens": 57102136 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-fable-5-1": { "accuracy": 90.263, "latency": 3459.994, "stderr": 1.566, "cost_per_test": 33.367507, "token_totals": { "input_tokens": 2858376432, "output_tokens": 11229337, "reasoning_tokens": 2013796, "cache_read_tokens": 2826345790, "cache_write_tokens": 32006248 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-6-astra": { "accuracy": 89.594, "latency": 2344.127, "stderr": 2.174, "cost_per_test": 38.512323, "token_totals": { "input_tokens": 813962842, "output_tokens": 7672087, "reasoning_tokens": 3013580, "cache_read_tokens": 762745769, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "openai/gpt-6.1-sol": { "accuracy": 88.931, "latency": 4489.755, "stderr": 1.893, "cost_per_test": 6.234794, "token_totals": { "input_tokens": 1073502468, "output_tokens": 10186788, "reasoning_tokens": 5383848, "cache_read_tokens": 1056860976, "cache_write_tokens": 16623336 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-opus-5": { "accuracy": 88.404, "latency": 5301.22, "stderr": 3.001, "cost_per_test": 33.876235, "token_totals": { "input_tokens": 2505984639, "output_tokens": 9909555, "reasoning_tokens": 1547105, "cache_read_tokens": 2470549366, "cache_write_tokens": 35417005 }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-6-sol": { "accuracy": 87.824, "latency": 2336.393, "stderr": 2.526, "cost_per_test": 26.356012, "token_totals": { "input_tokens": 3192235437, "output_tokens": 6823472, "reasoning_tokens": 2395580, "cache_read_tokens": 3162891655, "cache_write_tokens": 29258099 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.7": { "accuracy": 86.175, "latency": 2147.065, "stderr": 2.181, "cost_per_test": 15.82706, "token_totals": { "input_tokens": 767137889, "output_tokens": 7923865, "reasoning_tokens": 3838088, "cache_read_tokens": 713123456, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "meta/muse_spark_1_3_max": { "accuracy": 85.856, "latency": 977.668, "stderr": 2.508, "cost_per_test": 2.543084, "token_totals": { "input_tokens": 651739493, "output_tokens": 3048548, "reasoning_tokens": 422854, "cache_read_tokens": 636796800, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 85.223, "latency": 3646.574, "stderr": 3.392, "cost_per_test": 1.043846, "token_totals": { "input_tokens": 1076595168, "output_tokens": 7478091, "reasoning_tokens": 2852217, "cache_read_tokens": 979676672, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "kimi/kimi-k3": { "accuracy": 84.968, "latency": 999.021, "stderr": 2.746, "cost_per_test": 10.007576, "token_totals": { "input_tokens": 1261633418, "output_tokens": 5010801, "reasoning_tokens": 1248736, "cache_read_tokens": 1244327217, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 84.739, "latency": 928.101, "stderr": 2.891, "cost_per_test": 0.407445, "token_totals": { "input_tokens": 1429431621, "output_tokens": 7186472, "reasoning_tokens": 1913924, "cache_read_tokens": 1418642812, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "fireworks/ember-1": { "accuracy": 83.869, "latency": 2357.968, "stderr": 3.151, "cost_per_test": 8.559507, "token_totals": { "input_tokens": 981791343, "output_tokens": 4071290, "reasoning_tokens": 774425, "cache_read_tokens": 954988156, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands" }, "meta/muse_spark_1_3": { "accuracy": 82.859, "latency": 773.797, "stderr": 2.901, "cost_per_test": 2.101762, "token_totals": { "input_tokens": 401837405, "output_tokens": 2713902, "reasoning_tokens": 370079, "cache_read_tokens": 371584301, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "anthropic/claude-opus-4-8": { "accuracy": 82.725, "latency": 4549.256, "stderr": 3.081, "cost_per_test": 26.879989, "token_totals": { "input_tokens": 1802936139, "output_tokens": 9844882, "reasoning_tokens": 3416693, "cache_read_tokens": 1781691366, "cache_write_tokens": 21227618 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 82.297, "latency": 3982.6, "stderr": 3.148, "cost_per_test": 0.356023, "token_totals": { "input_tokens": 1830028483, "output_tokens": 7746431, "reasoning_tokens": 2336919, "cache_read_tokens": 1820296320, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-6-luna": { "accuracy": 81.649, "latency": 2219.515, "stderr": 3.377, "cost_per_test": 1.345703, "token_totals": { "input_tokens": 3186713261, "output_tokens": 9004550, "reasoning_tokens": 3632878, "cache_read_tokens": 3158117507, "cache_write_tokens": 28566327 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-sonnet-5": { "accuracy": 81.325, "latency": 4001.424, "stderr": 3.046, "cost_per_test": 25.38542, "token_totals": { "input_tokens": 5886886074, "output_tokens": 13734667, "reasoning_tokens": 5002326, "cache_read_tokens": 5849205195, "cache_write_tokens": 37648445 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-5.6-sol": { "accuracy": 80.495, "latency": 2033.047, "stderr": 3.716, "cost_per_test": 33.403634, "token_totals": { "input_tokens": 85484634, "output_tokens": 3878003, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse_spark_1_2": { "accuracy": 79.1, "latency": 1195.221, "stderr": 3.305, "cost_per_test": 1.529642, "token_totals": { "input_tokens": 342866386, "output_tokens": 2351182, "reasoning_tokens": 239369, "cache_read_tokens": 329175836, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 78.96, "latency": 3013.012, "stderr": 4.045, "cost_per_test": 0.563041, "token_totals": { "input_tokens": 1311689878, "output_tokens": 7947370, "reasoning_tokens": 3093850, "cache_read_tokens": 1157695808, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "google/gemini-3.8-flash": { "accuracy": 78.651, "latency": 519.104, "stderr": 3.884, "cost_per_test": 6.865141, "token_totals": { "input_tokens": 1149521706, "output_tokens": 6144190, "reasoning_tokens": 1439332, "cache_read_tokens": 1057116240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.3": { "accuracy": 78.125, "latency": 3847.197, "stderr": 3.94, "cost_per_test": 12.450185, "token_totals": { "input_tokens": 1971471587, "output_tokens": 8173329, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "anthropic/claude-opus-4-8-claude-code": { "accuracy": 77.485, "latency": 1142.14, "stderr": 3.737, "cost_per_test": 6.8557, "token_totals": { "input_tokens": 405940950, "output_tokens": 4014200, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "Claude Code" }, "tencent/hy4-preview": { "accuracy": 77.478, "latency": 2493.104, "stderr": 4.035, "cost_per_test": 2.181516, "token_totals": { "input_tokens": 1648623995, "output_tokens": 6263219, "reasoning_tokens": 1671679, "cache_read_tokens": 1618107200, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "OpenHands" }, "openai/gpt-5.6-luna": { "accuracy": 77.055, "latency": 1551.831, "stderr": 3.069, "cost_per_test": 0.726426, "token_totals": { "input_tokens": 54304210, "output_tokens": 3660213, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.6": { "accuracy": 76.239, "latency": 1530.145, "stderr": 3.824, "cost_per_test": 4.875513, "token_totals": { "input_tokens": 395097143, "output_tokens": 3804849, "reasoning_tokens": 775807, "cache_read_tokens": 379498496, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 74.736, "latency": 2241.572, "stderr": 3.442, "cost_per_test": 0.905592, "token_totals": { "input_tokens": 2153389585, "output_tokens": 7751888, "reasoning_tokens": 2696365, "cache_read_tokens": 2141888000, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-5.6-terra": { "accuracy": 74.594, "latency": 1152.433, "stderr": 4.203, "cost_per_test": 7.887015, "token_totals": { "input_tokens": 1192383214, "output_tokens": 4865371, "reasoning_tokens": 1379403, "cache_read_tokens": 1138222288, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse_spark_1_1": { "accuracy": 72.158, "latency": 615.428, "stderr": 3.327, "cost_per_test": 0.923283, "token_totals": { "input_tokens": 213516750, "output_tokens": 2218049, "reasoning_tokens": 189712, "cache_read_tokens": 182918822, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "anthropic/claude-opus-4-7": { "accuracy": 71.003, "latency": 2141.902, "stderr": 4.506, "cost_per_test": 21.407199, "token_totals": { "input_tokens": 1304465300, "output_tokens": 5421314, "reasoning_tokens": 277630, "cache_read_tokens": 1286180434, "cache_write_tokens": 18267241 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "stepfun/step-5-preview": { "accuracy": 70.399, "latency": 5466.358, "stderr": 4.267, "cost_per_test": 5.856257, "token_totals": { "input_tokens": 2504815838, "output_tokens": 8182685, "reasoning_tokens": 0, "cache_read_tokens": 2351680256, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "OpenHands" }, "google/gemini-3.7-flash": { "accuracy": 70.395, "latency": 1423.494, "stderr": 4.837, "cost_per_test": 4.827643, "token_totals": { "input_tokens": 911735026, "output_tokens": 5880420, "reasoning_tokens": 1013076, "cache_read_tokens": 866906330, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5.5": { "accuracy": 69.847, "latency": 1911.567, "stderr": 4.535, "cost_per_test": 16.661911, "token_totals": { "input_tokens": 92633221, "output_tokens": 3089898, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.5": { "accuracy": 69.002, "latency": 830.057, "stderr": 4.48, "cost_per_test": 4.102194, "token_totals": { "input_tokens": 264749332, "output_tokens": 3594207, "reasoning_tokens": 64753, "cache_read_tokens": 254635589, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 67.421, "latency": 5174.848, "stderr": 4.843, "cost_per_test": 10.686976, "token_totals": { "input_tokens": 90113056, "output_tokens": 5822598, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "openai/gpt-5.5-factory": { "accuracy": 67.391, "latency": 1177.43, "stderr": 4.75, "cost_per_test": 5.9268, "token_totals": { "input_tokens": 341996100, "output_tokens": 2443400, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Factory" }, "alibaba/qwen3.8-27b": { "accuracy": 64.85, "latency": 2619.407, "stderr": 4.825, "cost_per_test": 23.456372, "token_totals": { "input_tokens": 2292687899, "output_tokens": 8824879, "reasoning_tokens": 3211356, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "alibaba/qwen3.8-max": { "accuracy": 64.696, "latency": 10135.139, "stderr": 5.362, "cost_per_test": 8.238852, "token_totals": { "input_tokens": 1000088567, "output_tokens": 6067236, "reasoning_tokens": 2004745, "cache_read_tokens": 921541719, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "google/gemini-3.6-flash": { "accuracy": 64.005, "latency": 1523.201, "stderr": 4.29, "cost_per_test": 3.041993, "token_totals": { "input_tokens": 419948837, "output_tokens": 4065006, "reasoning_tokens": 715555, "cache_read_tokens": 376526772, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.2": { "accuracy": 63.961, "latency": 3748.167, "stderr": 4.795, "cost_per_test": 8.488514, "token_totals": { "input_tokens": 55483571, "output_tokens": 4081373, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "openai/gpt-5.3-codex": { "accuracy": 61.767, "latency": 4548.115, "stderr": 4.714, "cost_per_test": 11.914536, "token_totals": { "input_tokens": 192538209, "output_tokens": 4009383, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "openai/gpt-5.5-codex": { "accuracy": 58.194, "latency": 708.51, "stderr": 4.933, "cost_per_test": 3.6264, "token_totals": { "input_tokens": 167963850, "output_tokens": 2018400, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Codex" }, "anthropic/claude-opus-4-6": { "accuracy": 57.573, "latency": 1278.658, "stderr": 4.371, "cost_per_test": 8.687436, "token_totals": { "input_tokens": 8991, "output_tokens": 3314368, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-sonnet-4-6-claude-code": { "accuracy": 55.774, "latency": 2982.342, "stderr": 4.682, "cost_per_test": 7.354471, "token_totals": { "input_tokens": 896566178, "output_tokens": 362124, "reasoning_tokens": null, "cache_read_tokens": 869510565, "cache_write_tokens": 27028887 }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "Claude Code" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 53.499, "latency": 4971.341, "stderr": 5.067, "cost_per_test": 17.745182, "token_totals": { "input_tokens": 302477783, "output_tokens": 4406663, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 53.498, "latency": 1386.056, "stderr": 4.678, "cost_per_test": 8.279062, "token_totals": { "input_tokens": 8431, "output_tokens": 3390671, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-sonnet-4-6": { "accuracy": 51.476, "latency": 1572.118, "stderr": 4.645, "cost_per_test": 5.907892, "token_totals": { "input_tokens": 567692336, "output_tokens": 4210455, "reasoning_tokens": 442221, "cache_read_tokens": 557548000, "cache_write_tokens": 10135852 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro": { "accuracy": 49.931, "latency": 3418.92, "stderr": 4.767, "cost_per_test": 2.205991, "token_totals": { "input_tokens": 1767336432, "output_tokens": 7905241, "reasoning_tokens": 2425211, "cache_read_tokens": 1758006109, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "cursor/composer-2.5": { "accuracy": 49.614, "latency": 592.9, "stderr": 4.972, "cost_per_test": 0.502233, "token_totals": { "input_tokens": 94458506, "output_tokens": 2029275, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 200000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cursor", "harness": "Cursor CLI" }, "google/gemini-3.5-flash": { "accuracy": 48.683, "latency": 893.074, "stderr": 4.727, "cost_per_test": 2.540385, "token_totals": { "input_tokens": 64968703, "output_tokens": 3285132, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5.4": { "accuracy": 48.474, "latency": 1075.396, "stderr": 5.07, "cost_per_test": 7.50518, "token_totals": { "input_tokens": 428145333, "output_tokens": 3610264, "reasoning_tokens": 934026, "cache_read_tokens": 410377472, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Codex" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 47.969, "latency": 2055.117, "stderr": 5.606, "cost_per_test": 1.191299, "token_totals": { "input_tokens": 1529114850, "output_tokens": 14510964, "reasoning_tokens": 8384750, "cache_read_tokens": 1305150933, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "alibaba/qwen3.7-max": { "accuracy": 47.671, "latency": 4689.627, "stderr": 4.63, "cost_per_test": 11.400322, "token_totals": { "input_tokens": 197229291, "output_tokens": 2354762, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "minimax/MiniMax-M3": { "accuracy": 47.566, "latency": 5073.23, "stderr": 5.442, "cost_per_test": 6.453899, "token_totals": { "input_tokens": 61631839, "output_tokens": 5567819, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "kimi/kimi-k2.7-code": { "accuracy": 47.211, "latency": 10318.246, "stderr": 5.191, "cost_per_test": 3.835806, "token_totals": { "input_tokens": 12399493, "output_tokens": 3482721, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "alibaba/qwen3.7-plus": { "accuracy": 46.391, "latency": 2231.75, "stderr": 4.614, "cost_per_test": 1.082348, "token_totals": { "input_tokens": 55322964, "output_tokens": 3391824, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "xiaomi/mimo-v2.5": { "accuracy": 42.174, "latency": 1580.348, "stderr": 4.568, "cost_per_test": 0.068395, "token_totals": { "input_tokens": 7646791, "output_tokens": 2710586, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "openai/gpt-5.2-codex": { "accuracy": 37.912, "latency": 1934.792, "stderr": 4.576, "cost_per_test": 4.152495, "token_totals": { "input_tokens": 40226294, "output_tokens": 2658691, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "kimi/kimi-k2.6": { "accuracy": 37.891, "latency": 2967.768, "stderr": 4.912, "cost_per_test": 1.925351, "token_totals": { "input_tokens": 1291837961, "output_tokens": 3159307, "reasoning_tokens": 11486, "cache_read_tokens": 1269484743, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "google/gemini-3.5-flash-lite": { "accuracy": 37.161, "latency": 501.522, "stderr": 4.627, "cost_per_test": 0.833111, "token_totals": { "input_tokens": 387765124, "output_tokens": 3360015, "reasoning_tokens": 658865, "cache_read_tokens": 307681641, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "xiaomi/mimo-v2.5-pro": { "accuracy": 34.113, "latency": 2403.764, "stderr": 4.529, "cost_per_test": 0.166747, "token_totals": { "input_tokens": 7242687, "output_tokens": 3078284, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "google/gemini-3.1-pro-preview": { "accuracy": 32.034, "latency": 1211.906, "stderr": 4.342, "cost_per_test": 3.825202, "token_totals": { "input_tokens": 722761650, "output_tokens": 1941198, "reasoning_tokens": 260725, "cache_read_tokens": 678233584, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.1": { "accuracy": 31.456, "latency": 2014.732, "stderr": 4.553, "cost_per_test": 2.894911, "token_totals": { "input_tokens": 15170642, "output_tokens": 2734302, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "zai/glm-5.3-flash": { "accuracy": 30.759, "latency": 5027.281, "stderr": 5.277, "cost_per_test": 2.348805, "token_totals": { "input_tokens": 7308851703, "output_tokens": 11916569, "reasoning_tokens": null, "cache_read_tokens": 7228379520, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 26.097, "latency": 3276.224, "stderr": 5.075, "cost_per_test": 1.277298, "token_totals": { "input_tokens": 2524890518, "output_tokens": 10662405, "reasoning_tokens": 4838232, "cache_read_tokens": 2460443284, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "alibaba/qwen3.6-plus": { "accuracy": 25.565, "latency": 2289.977, "stderr": 3.906, "cost_per_test": 5.447636, "token_totals": { "input_tokens": 479493282, "output_tokens": 2653745, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 24.606, "latency": 1836.188, "stderr": 4.254, "cost_per_test": 2.574926, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "zai/glm-5-thinking": { "accuracy": 23.359, "latency": 13455.794, "stderr": 4.034, "cost_per_test": 40.270258, "token_totals": { "input_tokens": 1954171739, "output_tokens": 2870967, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 22.621, "latency": 2962.133, "stderr": 3.728, "cost_per_test": 6.656984, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-5.1-codex-max": { "accuracy": 22.168, "latency": 31182.938, "stderr": 3.84, "cost_per_test": 7.324105, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 20.63, "latency": 2283.142, "stderr": 3.159, "cost_per_test": 32.873283, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": "OpenHands" }, "google/gemini-3-flash-preview": { "accuracy": 20.204, "latency": 806.66, "stderr": 3.946, "cost_per_test": 0.942067, "token_totals": { "input_tokens": 317204132, "output_tokens": 2183311, "reasoning_tokens": 357653, "cache_read_tokens": 279403697, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5-2025-08-07": { "accuracy": 20.088, "latency": 1852.149, "stderr": 3.409, "cost_per_test": 1.525167, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse_spark": { "accuracy": 19.674, "latency": 981.298, "stderr": 3.966, "cost_per_test": 0.579705, "token_totals": { "input_tokens": 184911334, "output_tokens": 2080918, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "grok/grok-4.3": { "accuracy": 19.403, "latency": 589.407, "stderr": 4.243, "cost_per_test": 1.28472, "token_totals": { "input_tokens": 240295477, "output_tokens": 2046068, "reasoning_tokens": 220495, "cache_read_tokens": 235309534, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "thinkingmachines/inkling": { "accuracy": 19.214, "latency": 1105.492, "stderr": 3.659, "cost_per_test": 1.605949, "token_totals": { "input_tokens": 385384338, "output_tokens": 2166011, "reasoning_tokens": 0, "cache_read_tokens": 380152294, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "thinkingmachines/inkling-small": { "accuracy": 19.052, "latency": 2949.469, "stderr": 3.403, "cost_per_test": 0.432264, "token_totals": { "input_tokens": 294407624, "output_tokens": 2135670, "reasoning_tokens": null, "cache_read_tokens": 289327616, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "kimi/kimi-k2.5-thinking": { "accuracy": 17.536, "latency": 2570.381, "stderr": 3.258, "cost_per_test": 0.879292, "token_totals": { "input_tokens": 289848467, "output_tokens": 2490783, "reasoning_tokens": 123400, "cache_read_tokens": 283745267, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 15.738, "latency": 3015.618, "stderr": 3.179, "cost_per_test": 3.803761, "token_totals": { "input_tokens": 459410090, "output_tokens": 2440928, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "minimax/MiniMax-M2.5": { "accuracy": 14.853, "latency": 3065.622, "stderr": 2.953, "cost_per_test": 2.197729, "token_totals": { "input_tokens": 348550236, "output_tokens": 2628151, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "google/gemini-3-pro-preview": { "accuracy": 14.3, "latency": 10398.545, "stderr": 3.061, "cost_per_test": 6.417592, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 14.171, "latency": 698.246, "stderr": 3.539, "cost_per_test": 0.248587, "token_totals": { "input_tokens": 26079914, "output_tokens": 1087208, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-build-0.1": { "accuracy": 13.349, "latency": 394.638, "stderr": 3.819, "cost_per_test": null, "token_totals": { "input_tokens": 0, "output_tokens": 0, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Grok Build" }, "openai/gpt-5.1-codex": { "accuracy": 13.115, "latency": 3026.069, "stderr": 3.161, "cost_per_test": 3.797914, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "alibaba/qwen3.6-27b": { "accuracy": 11.941, "latency": 9762.856, "stderr": 3.414, "cost_per_test": 4.655127, "token_totals": { "input_tokens": 363862398, "output_tokens": 2524854, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "minimax/MiniMax-M2.7": { "accuracy": 11.926, "latency": 1377.326, "stderr": 2.857, "cost_per_test": 2.819109, "token_totals": { "input_tokens": 470065443, "output_tokens": 3235220, "reasoning_tokens": 171505, "cache_read_tokens": 63061859, "cache_write_tokens": 325666 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 11.393, "latency": 775.882, "stderr": 3.129, "cost_per_test": 1.306474, "token_totals": { "input_tokens": 1024173457, "output_tokens": 4124891, "reasoning_tokens": 878548, "cache_read_tokens": 1007363608, "cache_write_tokens": 16704885 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "poolside/laguna-m.1": { "accuracy": 11.038, "latency": 2381.926, "stderr": 2.822, "cost_per_test": 2.059401, "token_totals": { "input_tokens": 90126692, "output_tokens": 3112565, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": "OpenHands" }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 7.636, "latency": 2217.462, "stderr": 2.783, "cost_per_test": null, "token_totals": { "input_tokens": 1965727676, "output_tokens": 5440855, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": "OpenHands" }, "devin/swe-1-6-fast": { "accuracy": 7.22, "latency": 189.132, "stderr": 2.635, "cost_per_test": null, "token_totals": { "input_tokens": 0, "output_tokens": 0, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Devin", "harness": "Devin CLI" }, "poolside/laguna-xs.2": { "accuracy": 5.212, "latency": 2199.244, "stderr": 2.18, "cost_per_test": 0.981155, "token_totals": { "input_tokens": 312422657, "output_tokens": 2581908, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": "OpenHands" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 5.108, "latency": 3365.726, "stderr": 2.127, "cost_per_test": 2.469325, "token_totals": { "input_tokens": 21155716, "output_tokens": 2743130, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 4.063, "latency": 301.651, "stderr": 2.058, "cost_per_test": 0.771885, "token_totals": { "input_tokens": 7403442417, "output_tokens": 1887400, "reasoning_tokens": 76700, "cache_read_tokens": 7061849983, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "inception/mercury-2.5": { "accuracy": 3.683, "latency": 193.041, "stderr": 1.407, "cost_per_test": 0.378016, "token_totals": { "input_tokens": 198612247, "output_tokens": 1426011, "reasoning_tokens": 1298893, "cache_read_tokens": 121617600, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "OpenHands" }, "alibaba/qwen3-max": { "accuracy": 3.506, "latency": 3465.17, "stderr": 1.292, "cost_per_test": 6.023105, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 3.4, "latency": 1451.032, "stderr": 1.291, "cost_per_test": 0.233022, "token_totals": { "input_tokens": 666603077, "output_tokens": 7351064, "reasoning_tokens": null, "cache_read_tokens": 618088896, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": "OpenHands" }, "zai/glm-4.6": { "accuracy": 3.09, "latency": 10002.446, "stderr": 1.162, "cost_per_test": 10.845475, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "ant/ling-3.0-flash-2607": { "accuracy": 2.914, "latency": 1005.598, "stderr": 2.072, "cost_per_test": 0.292199, "token_totals": { "input_tokens": 803530511, "output_tokens": 3846224, "reasoning_tokens": null, "cache_read_tokens": 768376791, "cache_write_tokens": 0 }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": "OpenHands" }, "mistralai/mistral-medium-3.5": { "accuracy": 2.888, "latency": 2470.533, "stderr": 1.713, "cost_per_test": 16.735503, "token_totals": { "input_tokens": 538194513, "output_tokens": 3931115, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": "OpenHands" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 1.2, "latency": 527.562, "stderr": 1.2, "cost_per_test": 0.209444, "token_totals": { "input_tokens": 16252556, "output_tokens": 831827, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "google/gemini-2.5-pro": { "accuracy": 0.4, "latency": 2097.144, "stderr": 0.4, "cost_per_test": 1.223831, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 572.711, "stderr": 0.0, "cost_per_test": 0.51, "token_totals": { "input_tokens": 53939702, "output_tokens": 1042512, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 959.662, "stderr": 0.0, "cost_per_test": 0.128799, "token_totals": { "input_tokens": 535193452, "output_tokens": 3346611, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands" }, "grok/grok-4-fast-reasoning": { "accuracy": 0.0, "latency": 1448.855, "stderr": 0.0, "cost_per_test": 0.133206, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "mistralai/mistral-small-2603": { "accuracy": 0.0, "latency": 1814.681, "stderr": 0.0, "cost_per_test": 6.473001, "token_totals": { "input_tokens": 2145609935, "output_tokens": 3014270, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": "OpenHands" } } } }