{ "metadata": { "benchmark": "Vibe Code Bench v1.1", "slug": "vibe-code", "description": "Can models build web applications from scratch?", "benchmark_id": "vibe_code_bench", "family": "vibe_code_bench", "version": "1.1", "updated": "2026-11-09", "dataset_type": "private", "industry": "coding", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3-max", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "alibaba/qwen3.8-max-internal", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-fable-5-2", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-4-8-claude-code", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-4-6-claude-code", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "baseten/nemotron-3.5-super-vl-120b-a12b", "cursor/composer-2.5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "devin/swe-1-6-fast", "fireworks/deepseek-v3p2-thinking", "fireworks/ember-1", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "google/gemini-4-carbon", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "grok/grok-build-0.1", "inception/mercury-2.5", "inception/mercury-3", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k2.7-code", "kimi/kimi-k3", "kimi/kimi-k3.1-pro", "meta/muse-ember", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "minimax/MiniMax-M3.1-flash", "mistralai/mistral-large-4", "mistralai/mistral-large-4-oss", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2603", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.2-codex", "openai/gpt-5.3-codex", "openai/gpt-5.4", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.5-codex", "openai/gpt-5.5-factory", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "openai/gpt-6.1-terra", "poolside/laguna-m.1", "poolside/laguna-xs.2", "reflection/Beam-1.0", "stepfun/step-5-preview", "tencent/hy4", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-2-small", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-4.6", "zai/glm-5-thinking", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 50, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 127 }, "tasks": { "overall": { "anthropic/claude-fable-5-2": { "accuracy": 93.771, "latency": 2437.279, "stderr": 1.152, "cost_per_test": 22.13226, "token_totals": { "input_tokens": 1722388302, "output_tokens": 12878299, "reasoning_tokens": 3534023, "cache_read_tokens": 1697180927, "cache_write_tokens": 25192935 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 110.7, "avg_query_seconds": 22.0, "median_query_seconds": 9.4, "avg_ttft_seconds": 14.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": "OpenHands", "run_date": "2026-11-08" }, "anthropic/claude-sonnet-5-5": { "accuracy": 92.392, "latency": 3672.106, "stderr": 1.26, "cost_per_test": 19.729112, "token_totals": { "input_tokens": 5822927217, "output_tokens": 25491041, "reasoning_tokens": 11893204, "cache_read_tokens": 5760733058, "cache_write_tokens": 62167153 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 219.9, "avg_query_seconds": 16.7, "median_query_seconds": 7.8, "avg_ttft_seconds": 11.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands", "run_date": "2026-09-25" }, "google/gemini-4-argon": { "accuracy": 91.906, "latency": 2247.675, "stderr": 1.899, "cost_per_test": 8.10797, "token_totals": { "input_tokens": 451864686, "output_tokens": 5436370, "reasoning_tokens": 849287, "cache_read_tokens": 397575696, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 72.8, "avg_query_seconds": 30.9, "median_query_seconds": 13.9, "avg_ttft_seconds": 15.0, "output_time_split": { "reasoning_pct": 82.5, "content_pct": 0.0, "tool_call_pct": 17.5 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands", "run_date": "2026-09-24" }, "anthropic/claude-haiku-5-5": { "accuracy": 90.444, "latency": 2075.723, "stderr": 1.725, "cost_per_test": 6.071856, "token_totals": { "input_tokens": 4600663825, "output_tokens": 23863970, "reasoning_tokens": 13235182, "cache_read_tokens": 4559642564, "cache_write_tokens": 40996929 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 205.2, "avg_query_seconds": 10.1, "median_query_seconds": 4.6, "avg_ttft_seconds": 7.1, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands", "run_date": "2026-10-07" }, "anthropic/claude-fable-5": { "accuracy": 90.352, "latency": 3711.026, "stderr": 2.097, "cost_per_test": 41.714694, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-opus-5-5": { "accuracy": 90.294, "latency": 5766.524, "stderr": 1.526, "cost_per_test": 35.774773, "token_totals": { "input_tokens": 5596583425, "output_tokens": 19722501, "reasoning_tokens": 7457724, "cache_read_tokens": 5539450465, "cache_write_tokens": 57102136 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 231.0, "avg_query_seconds": 25.0, "median_query_seconds": 11.6, "avg_ttft_seconds": 18.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands", "run_date": "2026-09-18" }, "anthropic/claude-fable-5-1": { "accuracy": 90.263, "latency": 3459.994, "stderr": 1.566, "cost_per_test": 33.367507, "token_totals": { "input_tokens": 2858376432, "output_tokens": 11229337, "reasoning_tokens": 2013796, "cache_read_tokens": 2826345790, "cache_write_tokens": 32006248 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 191.7, "avg_query_seconds": 18.0, "median_query_seconds": 10.4, "avg_ttft_seconds": 10.8, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands", "run_date": "2026-08-29" }, "openai/gpt-6-astra": { "accuracy": 89.594, "latency": 2344.127, "stderr": 2.174, "cost_per_test": 38.512323, "token_totals": { "input_tokens": 813962842, "output_tokens": 7672087, "reasoning_tokens": 3013580, "cache_read_tokens": 762745769, "cache_write_tokens": null }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 106.1, "avg_query_seconds": 22.1, "median_query_seconds": 9.7, "avg_ttft_seconds": 11.5, "output_time_split": { "reasoning_pct": 8.0, "content_pct": 0.7, "tool_call_pct": 91.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands", "run_date": "2026-09-03" }, "google/gemini-4-carbon": { "accuracy": 89.06, "latency": 2172.019, "stderr": 2.294, "cost_per_test": 8.254926, "token_totals": { "input_tokens": 835863332, "output_tokens": 7234900, "reasoning_tokens": 1180038, "cache_read_tokens": 809317116, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 96.5, "avg_query_seconds": 22.5, "median_query_seconds": 11.4, "avg_ttft_seconds": 10.9, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands", "run_date": "2026-10-24" }, "openai/gpt-6.1-sol": { "accuracy": 88.931, "latency": 4489.755, "stderr": 1.893, "cost_per_test": 6.234794, "token_totals": { "input_tokens": 1073502468, "output_tokens": 10186788, "reasoning_tokens": 5383848, "cache_read_tokens": 1056860976, "cache_write_tokens": 16623336 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 121.0, "avg_query_seconds": 37.1, "median_query_seconds": 12.0, "avg_ttft_seconds": 26.4, "output_time_split": { "reasoning_pct": 6.5, "content_pct": 1.2, "tool_call_pct": 92.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands", "run_date": "2026-09-29" }, "anthropic/claude-opus-5": { "accuracy": 88.404, "latency": 5301.22, "stderr": 3.001, "cost_per_test": 33.876235, "token_totals": { "input_tokens": 2523942367, "output_tokens": 9766836, "reasoning_tokens": 1672106, "cache_read_tokens": 2491300194, "cache_write_tokens": 32623919 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-6-sol": { "accuracy": 87.824, "latency": 2336.393, "stderr": 2.526, "cost_per_test": 26.356012, "token_totals": { "input_tokens": 3192235437, "output_tokens": 6823472, "reasoning_tokens": 2395580, "cache_read_tokens": 3162891655, "cache_write_tokens": 29258099 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 212.7, "avg_query_seconds": 11.0, "median_query_seconds": 7.4, "avg_ttft_seconds": 5.8, "output_time_split": { "reasoning_pct": 18.1, "content_pct": 0.0, "tool_call_pct": 81.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands", "run_date": "2026-09-22" }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 87.73, "latency": 973.834, "stderr": 2.752, "cost_per_test": 0.579984, "token_totals": { "input_tokens": 2173326299, "output_tokens": 9246207, "reasoning_tokens": 3107112, "cache_read_tokens": 2156782720, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 170.2, "avg_query_seconds": 5.7, "median_query_seconds": 3.9, "avg_ttft_seconds": 2.2, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands", "run_date": "2026-10-17" }, "grok/grok-4.7": { "accuracy": 86.175, "latency": 2147.065, "stderr": 2.181, "cost_per_test": 15.82706, "token_totals": { "input_tokens": 767137889, "output_tokens": 7923865, "reasoning_tokens": 3838088, "cache_read_tokens": 713123456, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 86.4, "avg_query_seconds": 24.8, "median_query_seconds": 8.5, "avg_ttft_seconds": 3.4, "output_time_split": { "reasoning_pct": 75.7, "content_pct": 4.2, "tool_call_pct": 20.2 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands", "run_date": "2026-09-21" }, "meta/muse-wonder-full": { "accuracy": 85.996, "latency": 648.336, "stderr": 1.933, "cost_per_test": 2.293448, "token_totals": { "input_tokens": 222902077, "output_tokens": 2071146, "reasoning_tokens": 144351, "cache_read_tokens": 215357588, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 60.5, "avg_query_seconds": 10.7, "median_query_seconds": 5.5, "avg_ttft_seconds": 6.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands", "run_date": "2026-10-10" }, "meta/muse_spark_1_3_max": { "accuracy": 85.856, "latency": 977.668, "stderr": 2.508, "cost_per_test": 2.543084, "token_totals": { "input_tokens": 651739493, "output_tokens": 3048548, "reasoning_tokens": 422854, "cache_read_tokens": 636796800, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 118.0, "avg_query_seconds": 8.3, "median_query_seconds": 4.8, "avg_ttft_seconds": 7.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands", "run_date": "2026-09-05" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 85.223, "latency": 3646.574, "stderr": 3.392, "cost_per_test": 1.043846, "token_totals": { "input_tokens": 1076595168, "output_tokens": 7478091, "reasoning_tokens": 2852217, "cache_read_tokens": 979676672, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 111.4, "avg_query_seconds": 32.7, "median_query_seconds": 18.1, "avg_ttft_seconds": 13.8, "output_time_split": { "reasoning_pct": 36.4, "content_pct": 0.7, "tool_call_pct": 62.9 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands", "run_date": "2026-09-21" }, "kimi/kimi-k3": { "accuracy": 84.968, "latency": 999.021, "stderr": 2.746, "cost_per_test": 10.007576, "token_totals": { "input_tokens": 1261633418, "output_tokens": 5010801, "reasoning_tokens": 1248736, "cache_read_tokens": 1244327217, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 139.6, "avg_query_seconds": 7.2, "median_query_seconds": 6.2, "avg_ttft_seconds": 4.4, "output_time_split": { "reasoning_pct": 33.5, "content_pct": 2.5, "tool_call_pct": 64.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands", "run_date": "2026-10-01" }, "kimi/kimi-k3.1-pro": { "accuracy": 84.894, "latency": 3028.937, "stderr": 2.949, "cost_per_test": 7.485151, "token_totals": { "input_tokens": 1401107533, "output_tokens": 5806937, "reasoning_tokens": 1661626, "cache_read_tokens": 1381126038, "cache_write_tokens": 19409526 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 131.4, "avg_query_seconds": 23.1, "median_query_seconds": 14.0, "avg_ttft_seconds": 10.5, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands", "run_date": "2026-10-23" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 84.739, "latency": 928.101, "stderr": 2.891, "cost_per_test": 0.407445, "token_totals": { "input_tokens": 1429431621, "output_tokens": 7186472, "reasoning_tokens": 1913924, "cache_read_tokens": 1418642812, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 149.4, "avg_query_seconds": 6.2, "median_query_seconds": 4.9, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 36.8, "content_pct": 0.4, "tool_call_pct": 62.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands", "run_date": "2026-09-10" }, "openai/gpt-6.1-terra": { "accuracy": 83.997, "latency": 2250.002, "stderr": 2.865, "cost_per_test": 5.996953, "token_totals": { "input_tokens": 2454324986, "output_tokens": 5928692, "reasoning_tokens": 1847456, "cache_read_tokens": 2424165406, "cache_write_tokens": 30133951 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 170.9, "avg_query_seconds": 13.2, "median_query_seconds": 9.4, "avg_ttft_seconds": 7.0, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands", "run_date": "2026-10-26" }, "fireworks/ember-1": { "accuracy": 83.869, "latency": 2357.968, "stderr": 3.151, "cost_per_test": 8.559507, "token_totals": { "input_tokens": 981791343, "output_tokens": 4071290, "reasoning_tokens": 774425, "cache_read_tokens": 954988156, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 132.4, "avg_query_seconds": 17.8, "median_query_seconds": 10.4, "avg_ttft_seconds": 6.6, "output_time_split": { "reasoning_pct": 19.6, "content_pct": 1.9, "tool_call_pct": 78.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands", "run_date": "2026-09-28" }, "meta/muse-ember": { "accuracy": 83.703, "latency": 167.856, "stderr": 2.756, "cost_per_test": 0.054457, "token_totals": { "input_tokens": 267895132, "output_tokens": 1887915, "reasoning_tokens": 206822, "cache_read_tokens": 260527092, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 70.5, "avg_query_seconds": 2.4, "median_query_seconds": 1.9, "avg_ttft_seconds": 1.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 524288, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands", "run_date": "2026-11-03" }, "meta/muse-wonder": { "accuracy": 83.089, "latency": 350.076, "stderr": 2.49, "cost_per_test": 1.802723, "token_totals": { "input_tokens": 144518172, "output_tokens": 1830347, "reasoning_tokens": 114242, "cache_read_tokens": 138037760, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 52.3, "avg_query_seconds": 6.7, "median_query_seconds": 3.6, "avg_ttft_seconds": 4.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands", "run_date": "2026-10-10" }, "meta/muse_spark_1_3": { "accuracy": 82.859, "latency": 773.797, "stderr": 2.901, "cost_per_test": 2.101762, "token_totals": { "input_tokens": 401837405, "output_tokens": 2713902, "reasoning_tokens": 370079, "cache_read_tokens": 371584301, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 95.9, "avg_query_seconds": 8.1, "median_query_seconds": 5.4, "avg_ttft_seconds": 6.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands", "run_date": "2026-09-03" }, "anthropic/claude-opus-4-8": { "accuracy": 82.725, "latency": 4549.256, "stderr": 3.081, "cost_per_test": 26.879989, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 82.297, "latency": 3982.6, "stderr": 3.148, "cost_per_test": 0.356023, "token_totals": { "input_tokens": 1830028483, "output_tokens": 7746431, "reasoning_tokens": 2336919, "cache_read_tokens": 1820296320, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-6-luna": { "accuracy": 81.649, "latency": 2219.515, "stderr": 3.377, "cost_per_test": 1.345703, "token_totals": { "input_tokens": 3186713261, "output_tokens": 9004550, "reasoning_tokens": 3632878, "cache_read_tokens": 3158117507, "cache_write_tokens": 28566327 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 196.2, "avg_query_seconds": 11.3, "median_query_seconds": 7.6, "avg_ttft_seconds": 5.5, "output_time_split": { "reasoning_pct": 28.4, "content_pct": 0.0, "tool_call_pct": 71.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands", "run_date": "2026-09-22" }, "mistralai/mistral-large-4-oss": { "accuracy": 81.555, "latency": 3012.646, "stderr": 3.289, "cost_per_test": 11.723179, "token_totals": { "input_tokens": 1215599434, "output_tokens": 6354662, "reasoning_tokens": null, "cache_read_tokens": 441437184, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 131.9, "avg_query_seconds": 22.8, "median_query_seconds": 20.7, "avg_ttft_seconds": 14.6, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": "OpenHands", "run_date": "2026-10-26" }, "thinkingmachines/inkling-2-small": { "accuracy": 81.546, "latency": 1537.412, "stderr": 3.454, "cost_per_test": 1.901278, "token_totals": { "input_tokens": 2544521097, "output_tokens": 9443332, "reasoning_tokens": null, "cache_read_tokens": 2510132352, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 173.8, "avg_query_seconds": 8.8, "median_query_seconds": 6.3, "avg_ttft_seconds": 5.3, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands", "run_date": "2026-11-03" }, "anthropic/claude-sonnet-5": { "accuracy": 81.325, "latency": 4001.424, "stderr": 3.046, "cost_per_test": 25.38542, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "tencent/hy4": { "accuracy": 81.008, "latency": 2281.498, "stderr": 3.548, "cost_per_test": 2.394585, "token_totals": { "input_tokens": 1730119557, "output_tokens": 7711952, "reasoning_tokens": 3236621, "cache_read_tokens": 1695048064, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 165.4, "avg_query_seconds": 13.8, "median_query_seconds": 9.8, "avg_ttft_seconds": 6.3, "output_time_split": null }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "OpenHands", "run_date": "2026-11-04" }, "openai/gpt-5.6-sol": { "accuracy": 80.495, "latency": 2033.047, "stderr": 3.716, "cost_per_test": 33.403634, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse_spark_1_2": { "accuracy": 79.1, "latency": 1195.221, "stderr": 3.305, "cost_per_test": 1.529642, "token_totals": { "input_tokens": 342866386, "output_tokens": 2351182, "reasoning_tokens": 239369, "cache_read_tokens": 329175836, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 78.96, "latency": 3013.012, "stderr": 4.045, "cost_per_test": 0.563041, "token_totals": { "input_tokens": 1311689878, "output_tokens": 7947370, "reasoning_tokens": 3093850, "cache_read_tokens": 1157695808, "cache_write_tokens": 0 }, "token_tasks_measured": 48, "turn_metrics": { "tasks_measured": 48, "avg_turns": 122.0, "avg_query_seconds": 24.7, "median_query_seconds": 14.8, "avg_ttft_seconds": 13.5, "output_time_split": { "reasoning_pct": 32.7, "content_pct": 0.6, "tool_call_pct": 66.7 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands", "run_date": "2026-09-21" }, "google/gemini-3.8-flash": { "accuracy": 78.651, "latency": 519.104, "stderr": 3.884, "cost_per_test": 6.865141, "token_totals": { "input_tokens": 1149521706, "output_tokens": 6144190, "reasoning_tokens": 1439332, "cache_read_tokens": 1057116240, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 146.1, "avg_query_seconds": 3.6, "median_query_seconds": 2.8, "avg_ttft_seconds": 2.6, "output_time_split": { "reasoning_pct": 96.1, "content_pct": 0.0, "tool_call_pct": 3.9 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands", "run_date": "2026-08-27" }, "mistralai/mistral-large-4": { "accuracy": 78.403, "latency": 10395.145, "stderr": 3.55, "cost_per_test": 20.281954, "token_totals": { "input_tokens": 1823486517, "output_tokens": 8065079, "reasoning_tokens": null, "cache_read_tokens": 443300864, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 183.4, "avg_query_seconds": 56.7, "median_query_seconds": 45.1, "avg_ttft_seconds": 29.9, "output_time_split": { "reasoning_pct": 66.5, "content_pct": 33.4, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": "OpenHands", "run_date": "2026-10-04" }, "zai/glm-5.3": { "accuracy": 78.125, "latency": 3847.197, "stderr": 3.94, "cost_per_test": 12.450185, "token_totals": { "input_tokens": 1971471587, "output_tokens": 8173329, "reasoning_tokens": null, "cache_read_tokens": 1906590911, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 194.4, "avg_query_seconds": 19.8, "median_query_seconds": 8.0, "avg_ttft_seconds": 11.9, "output_time_split": { "reasoning_pct": 42.9, "content_pct": 1.1, "tool_call_pct": 56.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands", "run_date": "2026-08-28" }, "anthropic/claude-opus-4-8-claude-code": { "accuracy": 77.485, "latency": 1142.14, "stderr": 3.737, "cost_per_test": 6.8557, "token_totals": { "input_tokens": 405940950, "output_tokens": 4014200, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "Claude Code" }, "tencent/hy4-preview": { "accuracy": 77.478, "latency": 2493.104, "stderr": 4.035, "cost_per_test": 2.181516, "token_totals": { "input_tokens": 1648623995, "output_tokens": 6263219, "reasoning_tokens": 1671679, "cache_read_tokens": 1618107200, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 178.5, "avg_query_seconds": 14.0, "median_query_seconds": 8.9, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 31.8, "content_pct": 0.8, "tool_call_pct": 67.4 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "OpenHands", "run_date": "2026-09-15" }, "alibaba/qwen3.8-max-internal": { "accuracy": 77.141, "latency": 5079.378, "stderr": 3.721, "cost_per_test": 11.668329, "token_totals": { "input_tokens": 1895650724, "output_tokens": 8327720, "reasoning_tokens": 2363284, "cache_read_tokens": 1861629312, "cache_write_tokens": null }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 171.6, "avg_query_seconds": 29.6, "median_query_seconds": 18.6, "avg_ttft_seconds": 17.6, "output_time_split": { "reasoning_pct": 33.2, "content_pct": 0.3, "tool_call_pct": 66.5 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands", "run_date": "2026-10-12" }, "openai/gpt-5.6-luna": { "accuracy": 77.055, "latency": 1551.831, "stderr": 3.069, "cost_per_test": 0.726426, "token_totals": { "input_tokens": 54304210, "output_tokens": 3660213, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.6": { "accuracy": 76.239, "latency": 1530.145, "stderr": 3.824, "cost_per_test": 4.875513, "token_totals": { "input_tokens": 395097143, "output_tokens": 3804849, "reasoning_tokens": 775807, "cache_read_tokens": 379498496, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "meta/muse-spark-open": { "accuracy": 76.187, "latency": 1869.172, "stderr": 3.317, "cost_per_test": 1.655609, "token_totals": { "input_tokens": 225685852, "output_tokens": 2085879, "reasoning_tokens": 952456, "cache_read_tokens": 189265312, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 70.3, "avg_query_seconds": 26.6, "median_query_seconds": 6.8, "avg_ttft_seconds": 6.7, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": "OpenHands", "run_date": "2026-10-15" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 74.736, "latency": 2241.572, "stderr": 3.442, "cost_per_test": 0.905592, "token_totals": { "input_tokens": 2153389585, "output_tokens": 7751888, "reasoning_tokens": 2696365, "cache_read_tokens": 2141888000, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-5.6-terra": { "accuracy": 74.594, "latency": 1152.433, "stderr": 4.203, "cost_per_test": 7.887015, "token_totals": { "input_tokens": 1192383214, "output_tokens": 4865371, "reasoning_tokens": 1379403, "cache_read_tokens": 1138222288, "cache_write_tokens": null }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 127.4, "avg_query_seconds": 9.0, "median_query_seconds": 5.4, "avg_ttft_seconds": 4.1, "output_time_split": { "reasoning_pct": 23.9, "content_pct": 0.0, "tool_call_pct": 76.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands", "run_date": "2026-08-14" }, "minimax/MiniMax-M3.1-flash": { "accuracy": 73.944, "latency": 1263.916, "stderr": 4.249, "cost_per_test": 1.146649, "token_totals": { "input_tokens": 3334725307, "output_tokens": 8488142, "reasoning_tokens": 2462335, "cache_read_tokens": 3318290676, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 231.6, "avg_query_seconds": 5.5, "median_query_seconds": 4.5, "avg_ttft_seconds": 3.0, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": "OpenHands", "run_date": "2026-10-16" }, "meta/muse_spark_1_1": { "accuracy": 72.158, "latency": 615.428, "stderr": 3.327, "cost_per_test": 0.923283, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "anthropic/claude-opus-4-7": { "accuracy": 71.003, "latency": 2141.902, "stderr": 4.506, "cost_per_test": 21.407199, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "stepfun/step-5-preview": { "accuracy": 70.399, "latency": 5466.358, "stderr": 4.267, "cost_per_test": 5.856257, "token_totals": { "input_tokens": 2504815838, "output_tokens": 8182685, "reasoning_tokens": 0, "cache_read_tokens": 2351680256, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 231.7, "avg_query_seconds": 23.6, "median_query_seconds": 18.0, "avg_ttft_seconds": 14.0, "output_time_split": { "reasoning_pct": 38.8, "content_pct": 2.1, "tool_call_pct": 59.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "OpenHands", "run_date": "2026-10-02" }, "google/gemini-3.7-flash": { "accuracy": 70.395, "latency": 1423.494, "stderr": 4.837, "cost_per_test": 4.827643, "token_totals": { "input_tokens": 911735026, "output_tokens": 5880420, "reasoning_tokens": 1013076, "cache_read_tokens": 866906330, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5.5": { "accuracy": 69.847, "latency": 1911.567, "stderr": 4.535, "cost_per_test": 16.661911, "token_totals": { "input_tokens": 92633221, "output_tokens": 3089898, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.5": { "accuracy": 69.002, "latency": 830.057, "stderr": 4.48, "cost_per_test": 4.102194, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 67.421, "latency": 5174.848, "stderr": 4.843, "cost_per_test": 10.686976, "token_totals": { "input_tokens": 90113056, "output_tokens": 5822598, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "openai/gpt-5.5-factory": { "accuracy": 67.391, "latency": 1177.43, "stderr": 4.75, "cost_per_test": 5.9268, "token_totals": { "input_tokens": 341996100, "output_tokens": 2443400, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Factory" }, "reflection/Beam-1.0": { "accuracy": 66.436, "latency": 145.263, "stderr": 4.062, "cost_per_test": null, "token_totals": { "input_tokens": 430528251, "output_tokens": 2004844, "reasoning_tokens": 550281, "cache_read_tokens": 419321600, "cache_write_tokens": null }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 129.5, "avg_query_seconds": 1.1, "median_query_seconds": 0.6, "avg_ttft_seconds": 0.5, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": "OpenHands", "run_date": "2026-10-16" }, "alibaba/qwen3.8-27b": { "accuracy": 64.85, "latency": 2619.407, "stderr": 4.825, "cost_per_test": 23.456372, "token_totals": { "input_tokens": 2292687899, "output_tokens": 8824879, "reasoning_tokens": 3211356, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 203.2, "avg_query_seconds": 12.9, "median_query_seconds": 10.3, "avg_ttft_seconds": 7.2, "output_time_split": { "reasoning_pct": 45.6, "content_pct": 1.9, "tool_call_pct": 52.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands", "run_date": "2026-08-17" }, "alibaba/qwen3.8-max": { "accuracy": 64.696, "latency": 10135.139, "stderr": 5.362, "cost_per_test": 8.238852, "token_totals": { "input_tokens": 966988648, "output_tokens": 5955904, "reasoning_tokens": 1971427, "cache_read_tokens": 890154368, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "google/gemini-3.6-flash": { "accuracy": 64.005, "latency": 1523.201, "stderr": 4.29, "cost_per_test": 3.041993, "token_totals": { "input_tokens": 419948837, "output_tokens": 4065006, "reasoning_tokens": 715555, "cache_read_tokens": 376526772, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.2": { "accuracy": 63.961, "latency": 3748.167, "stderr": 4.795, "cost_per_test": 8.488514, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "openai/gpt-5.3-codex": { "accuracy": 61.767, "latency": 4548.115, "stderr": 4.714, "cost_per_test": 11.914536, "token_totals": { "input_tokens": 192538209, "output_tokens": 4009383, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "inception/mercury-3": { "accuracy": 60.927, "latency": 566.58, "stderr": 5.379, "cost_per_test": 0.627191, "token_totals": { "input_tokens": 1422007111, "output_tokens": 6493005, "reasoning_tokens": 5135055, "cache_read_tokens": 1256819130, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 171.6, "avg_query_seconds": 3.3, "median_query_seconds": 2.2, "avg_ttft_seconds": 2.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "OpenHands", "run_date": "2026-10-19" }, "openai/gpt-5.5-codex": { "accuracy": 58.194, "latency": 708.51, "stderr": 4.933, "cost_per_test": 3.6264, "token_totals": { "input_tokens": 167963850, "output_tokens": 2018400, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Codex" }, "anthropic/claude-opus-4-6": { "accuracy": 57.573, "latency": 1278.658, "stderr": 4.371, "cost_per_test": 8.687436, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-sonnet-4-6-claude-code": { "accuracy": 55.774, "latency": 2982.342, "stderr": 4.682, "cost_per_test": 7.354471, "token_totals": { "input_tokens": 896566178, "output_tokens": 362124, "reasoning_tokens": null, "cache_read_tokens": 869510565, "cache_write_tokens": 27028887 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "Claude Code" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 53.499, "latency": 4971.341, "stderr": 5.067, "cost_per_test": 17.745182, "token_totals": { "input_tokens": 302477783, "output_tokens": 4406663, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 53.498, "latency": 1386.056, "stderr": 4.678, "cost_per_test": 8.279062, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-sonnet-4-6": { "accuracy": 51.476, "latency": 1572.118, "stderr": 4.645, "cost_per_test": 5.907892, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro": { "accuracy": 49.931, "latency": 3418.92, "stderr": 4.767, "cost_per_test": 2.205991, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "cursor/composer-2.5": { "accuracy": 49.614, "latency": 592.9, "stderr": 4.972, "cost_per_test": 0.502233, "token_totals": { "input_tokens": 94458506, "output_tokens": 2029275, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 200000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cursor", "harness": "Cursor CLI" }, "google/gemini-3.5-flash": { "accuracy": 48.683, "latency": 893.074, "stderr": 4.727, "cost_per_test": 2.540385, "token_totals": { "input_tokens": 64968703, "output_tokens": 3285132, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5.4": { "accuracy": 48.474, "latency": 1075.396, "stderr": 5.07, "cost_per_test": 7.50518, "token_totals": { "input_tokens": 428145333, "output_tokens": 3610264, "reasoning_tokens": 934026, "cache_read_tokens": 410377472, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Codex" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 47.969, "latency": 2055.117, "stderr": 5.606, "cost_per_test": 1.191299, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "alibaba/qwen3.7-max": { "accuracy": 47.671, "latency": 4689.627, "stderr": 4.63, "cost_per_test": 11.400322, "token_totals": { "input_tokens": 197229291, "output_tokens": 2354762, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "minimax/MiniMax-M3": { "accuracy": 47.566, "latency": 5073.23, "stderr": 5.442, "cost_per_test": 6.453899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "kimi/kimi-k2.7-code": { "accuracy": 47.211, "latency": 10318.246, "stderr": 5.191, "cost_per_test": 3.835806, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "alibaba/qwen3.7-plus": { "accuracy": 46.391, "latency": 2231.75, "stderr": 4.614, "cost_per_test": 1.082348, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "xiaomi/mimo-v2.5": { "accuracy": 42.174, "latency": 1580.348, "stderr": 4.568, "cost_per_test": 0.068395, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "openai/gpt-5.2-codex": { "accuracy": 37.912, "latency": 1934.792, "stderr": 4.576, "cost_per_test": 4.152495, "token_totals": { "input_tokens": 40226294, "output_tokens": 2658691, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "kimi/kimi-k2.6": { "accuracy": 37.891, "latency": 2967.768, "stderr": 4.912, "cost_per_test": 1.925351, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "google/gemini-3.5-flash-lite": { "accuracy": 37.161, "latency": 501.522, "stderr": 4.627, "cost_per_test": 0.833111, "token_totals": { "input_tokens": 387765124, "output_tokens": 3360015, "reasoning_tokens": 658865, "cache_read_tokens": 307681641, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "xiaomi/mimo-v2.5-pro": { "accuracy": 34.113, "latency": 2403.764, "stderr": 4.529, "cost_per_test": 0.166747, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "google/gemini-3.1-pro-preview": { "accuracy": 32.034, "latency": 1211.906, "stderr": 4.342, "cost_per_test": 3.825202, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.1": { "accuracy": 31.456, "latency": 2014.732, "stderr": 4.553, "cost_per_test": 2.894911, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "zai/glm-5.3-flash": { "accuracy": 30.759, "latency": 5027.281, "stderr": 5.277, "cost_per_test": 4.69761, "token_totals": { "input_tokens": 7308851703, "output_tokens": 11916569, "reasoning_tokens": null, "cache_read_tokens": 7228379520, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 556.0, "avg_query_seconds": 9.0, "median_query_seconds": 5.5, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 37.2, "content_pct": 1.7, "tool_call_pct": 61.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands", "run_date": "2026-08-28" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 26.097, "latency": 3276.224, "stderr": 5.075, "cost_per_test": 1.277298, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "alibaba/qwen3.6-plus": { "accuracy": 25.565, "latency": 2289.977, "stderr": 3.906, "cost_per_test": 5.447636, "token_totals": { "input_tokens": 479493282, "output_tokens": 2653745, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 24.606, "latency": 1836.188, "stderr": 4.254, "cost_per_test": 2.574926, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "zai/glm-5-thinking": { "accuracy": 23.359, "latency": 13455.794, "stderr": 4.034, "cost_per_test": 40.270258, "token_totals": { "input_tokens": 1954171739, "output_tokens": 2870967, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 22.621, "latency": 2962.133, "stderr": 3.728, "cost_per_test": 6.656984, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-5.1-codex-max": { "accuracy": 22.168, "latency": 31182.938, "stderr": 3.84, "cost_per_test": 7.324105, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 20.63, "latency": 2283.142, "stderr": 3.159, "cost_per_test": 32.873283, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": "OpenHands" }, "google/gemini-3-flash-preview": { "accuracy": 20.204, "latency": 806.66, "stderr": 3.946, "cost_per_test": 0.942067, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5-2025-08-07": { "accuracy": 20.088, "latency": 1852.149, "stderr": 3.409, "cost_per_test": 1.525167, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse_spark": { "accuracy": 19.674, "latency": 981.298, "stderr": 3.966, "cost_per_test": 0.579705, "token_totals": { "input_tokens": 184911334, "output_tokens": 2080918, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "grok/grok-4.3": { "accuracy": 19.403, "latency": 589.407, "stderr": 4.243, "cost_per_test": 1.28472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "thinkingmachines/inkling": { "accuracy": 19.214, "latency": 1105.492, "stderr": 3.659, "cost_per_test": 1.605949, "token_totals": { "input_tokens": 391977485, "output_tokens": 2288277, "reasoning_tokens": 0, "cache_read_tokens": 386683776, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "thinkingmachines/inkling-small": { "accuracy": 19.052, "latency": 2949.469, "stderr": 3.403, "cost_per_test": 0.432264, "token_totals": { "input_tokens": 294407624, "output_tokens": 2135670, "reasoning_tokens": null, "cache_read_tokens": 289327616, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "kimi/kimi-k2.5-thinking": { "accuracy": 17.536, "latency": 2570.381, "stderr": 3.258, "cost_per_test": 0.879292, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 15.738, "latency": 3015.618, "stderr": 3.179, "cost_per_test": 3.803761, "token_totals": { "input_tokens": 459410090, "output_tokens": 2440928, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "minimax/MiniMax-M2.5": { "accuracy": 14.853, "latency": 3065.622, "stderr": 2.953, "cost_per_test": 2.197729, "token_totals": { "input_tokens": 348550236, "output_tokens": 2628151, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "google/gemini-3-pro-preview": { "accuracy": 14.3, "latency": 10398.545, "stderr": 3.061, "cost_per_test": 6.417592, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 14.171, "latency": 698.246, "stderr": 3.539, "cost_per_test": 0.248587, "token_totals": { "input_tokens": 26079914, "output_tokens": 1087208, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-build-0.1": { "accuracy": 13.349, "latency": 394.638, "stderr": 3.819, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Grok Build" }, "openai/gpt-5.1-codex": { "accuracy": 13.115, "latency": 3026.069, "stderr": 3.161, "cost_per_test": 3.797914, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "alibaba/qwen3.6-27b": { "accuracy": 11.941, "latency": 9762.856, "stderr": 3.414, "cost_per_test": 4.655127, "token_totals": { "input_tokens": 363862398, "output_tokens": 2524854, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 49, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "minimax/MiniMax-M2.7": { "accuracy": 11.926, "latency": 1377.326, "stderr": 2.857, "cost_per_test": 2.819109, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 11.393, "latency": 775.882, "stderr": 3.129, "cost_per_test": 1.306474, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "poolside/laguna-m.1": { "accuracy": 11.038, "latency": 2381.926, "stderr": 2.822, "cost_per_test": 2.059401, "token_totals": { "input_tokens": 90126692, "output_tokens": 3112565, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": "OpenHands" }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 7.636, "latency": 2217.462, "stderr": 2.783, "cost_per_test": null, "token_totals": { "input_tokens": 1965727676, "output_tokens": 5440855, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": "OpenHands" }, "devin/swe-1-6-fast": { "accuracy": 7.22, "latency": 189.132, "stderr": 2.635, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Devin", "harness": "Devin CLI" }, "poolside/laguna-xs.2": { "accuracy": 5.212, "latency": 2199.244, "stderr": 2.18, "cost_per_test": 0.981155, "token_totals": { "input_tokens": 312422657, "output_tokens": 2581908, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": "OpenHands" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 5.108, "latency": 3365.726, "stderr": 2.127, "cost_per_test": 2.469325, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 4.063, "latency": 301.651, "stderr": 2.058, "cost_per_test": 0.771885, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 3.71, "latency": 262.905, "stderr": 1.926, "cost_per_test": 0.539233, "token_totals": { "input_tokens": 419784036, "output_tokens": 2901901, "reasoning_tokens": 402890, "cache_read_tokens": 365288192, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 116.8, "avg_query_seconds": 2.3, "median_query_seconds": 1.3, "avg_ttft_seconds": 1.5, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": "OpenHands", "run_date": "2026-10-28" }, "inception/mercury-2.5": { "accuracy": 3.683, "latency": 193.041, "stderr": 1.407, "cost_per_test": 0.378016, "token_totals": { "input_tokens": 198612247, "output_tokens": 1426011, "reasoning_tokens": 1298893, "cache_read_tokens": 121617600, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 85.0, "avg_query_seconds": 2.3, "median_query_seconds": 1.6, "avg_ttft_seconds": 1.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "OpenHands", "run_date": "2026-09-10" }, "alibaba/qwen3-max": { "accuracy": 3.506, "latency": 3465.17, "stderr": 1.292, "cost_per_test": 6.023105, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 3.4, "latency": 1451.032, "stderr": 1.291, "cost_per_test": 0.233022, "token_totals": { "input_tokens": 666603077, "output_tokens": 7351064, "reasoning_tokens": null, "cache_read_tokens": 618088896, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 114.0, "avg_query_seconds": 12.7, "median_query_seconds": 5.0, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 7.1, "content_pct": 53.9, "tool_call_pct": 38.9 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": "OpenHands", "run_date": "2026-09-17" }, "zai/glm-4.6": { "accuracy": 3.09, "latency": 10002.446, "stderr": 1.162, "cost_per_test": 10.845475, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "ant/ling-3.0-flash-2607": { "accuracy": 2.914, "latency": 1005.598, "stderr": 2.072, "cost_per_test": 0.292199, "token_totals": { "input_tokens": 779810749, "output_tokens": 3806141, "reasoning_tokens": null, "cache_read_tokens": 745220087, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": "OpenHands" }, "mistralai/mistral-medium-3.5": { "accuracy": 2.888, "latency": 2470.533, "stderr": 1.713, "cost_per_test": 16.735503, "token_totals": { "input_tokens": 538194513, "output_tokens": 3931115, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": "OpenHands" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 1.2, "latency": 527.562, "stderr": 1.2, "cost_per_test": 0.209444, "token_totals": { "input_tokens": 16252556, "output_tokens": 831827, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "google/gemini-2.5-pro": { "accuracy": 0.4, "latency": 2097.144, "stderr": 0.4, "cost_per_test": 1.223831, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 572.711, "stderr": 0.0, "cost_per_test": 0.51, "token_totals": { "input_tokens": 53939702, "output_tokens": 1042512, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 959.662, "stderr": 0.0, "cost_per_test": 0.128799, "token_totals": { "input_tokens": 535193452, "output_tokens": 3346611, "reasoning_tokens": null, "cache_read_tokens": 524725810, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 144.2, "avg_query_seconds": 6.7, "median_query_seconds": 1.5, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 33.5, "content_pct": 33.6, "tool_call_pct": 32.8 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands", "run_date": "2026-08-11" }, "grok/grok-4-fast-reasoning": { "accuracy": 0.0, "latency": 1448.855, "stderr": 0.0, "cost_per_test": 0.133206, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "mistralai/mistral-small-2603": { "accuracy": 0.0, "latency": 1814.681, "stderr": 0.0, "cost_per_test": 6.473001, "token_totals": { "input_tokens": 2145609935, "output_tokens": 3014270, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": "OpenHands" } } } }