{ "metadata": { "benchmark": "Robo-bench", "slug": "robo_bench", "description": "Can a model control a robot arm and a humanoid in a real-time physics simulation, from motor commands and camera images alone? Most tasks are held out.", "benchmark_id": "robo_bench", "family": "robo_bench", "version": "1", "updated": "2026-10-13", "dataset_type": "private", "industry": "science", "tasks": { "overall": "Overall", "arm": "Arm (13 tasks)", "humanoid": "Humanoid (11 tasks)", "balance_scale": "Balance scale (public)", "lego_manual": "Lego manual (public)", "ninja_course": "Ninja course (public)" }, "models": [ "anthropic/claude-fable-5-1", "anthropic/claude-opus-5-5", "google/gemini-4-argon", "grok/grok-4.7", "meta/muse-wonder-full", "openai/gpt-6-astra", "openai/gpt-6.1-sol" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "cost_label": "Est. API cost", "show_stderr": false, "latency_label": "Time to first move", "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 7 }, "tasks": { "overall": { "openai/gpt-6.1-sol": { "accuracy": 28.333, "latency": 79.89, "stderr": 0.0, "cost_per_test": 4.9675, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "openai/gpt-6-astra": { "accuracy": 23.333, "latency": 74.88, "stderr": 0.0, "cost_per_test": 28.479, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "anthropic/claude-opus-5-5": { "accuracy": 14.583, "latency": 127.15, "stderr": 0.0, "cost_per_test": 11.3832, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "meta/muse-wonder-full": { "accuracy": 10.417, "latency": 133.94, "stderr": 0.0, "cost_per_test": 19.6767, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "meta", "harness": "muse-code" }, "anthropic/claude-fable-5-1": { "accuracy": 5.833, "latency": 124.05, "stderr": 0.0, "cost_per_test": 20.4045, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "google/gemini-4-argon": { "accuracy": 0.833, "latency": 152.43, "stderr": 0.0, "cost_per_test": 17.332013, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "high", "verbosity": null, "compute_effort": "high", "provider": "google", "harness": "antigravity" }, "grok/grok-4.7": { "accuracy": 0.208, "latency": 150.61, "stderr": 0.0, "cost_per_test": 11.2507, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": "xhigh", "provider": "grok", "harness": "grok-build" } }, "arm": { "openai/gpt-6.1-sol": { "accuracy": 50.769, "latency": 59.02, "stderr": 0.0, "cost_per_test": 2.4062, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "openai/gpt-6-astra": { "accuracy": 42.308, "latency": 57.72, "stderr": 0.0, "cost_per_test": 14.3751, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "anthropic/claude-opus-5-5": { "accuracy": 25.385, "latency": 140.16, "stderr": 0.0, "cost_per_test": 6.6251, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "meta/muse-wonder-full": { "accuracy": 17.692, "latency": 145.08, "stderr": 0.0, "cost_per_test": 10.0103, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "meta", "harness": "muse-code" }, "anthropic/claude-fable-5-1": { "accuracy": 10.769, "latency": 163.46, "stderr": 0.0, "cost_per_test": 12.4198, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "google/gemini-4-argon": { "accuracy": 1.538, "latency": 140.76, "stderr": 0.0, "cost_per_test": 8.628905, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "high", "verbosity": null, "compute_effort": "high", "provider": "google", "harness": "antigravity" }, "grok/grok-4.7": { "accuracy": 0.385, "latency": 147.99, "stderr": 0.0, "cost_per_test": 6.5501, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": "xhigh", "provider": "grok", "harness": "grok-build" } }, "humanoid": { "openai/gpt-6.1-sol": { "accuracy": 1.818, "latency": 104.55, "stderr": 0.0, "cost_per_test": 2.5613, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "anthropic/claude-opus-5-5": { "accuracy": 1.818, "latency": 111.78, "stderr": 0.0, "cost_per_test": 4.7581, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "meta/muse-wonder-full": { "accuracy": 1.818, "latency": 120.78, "stderr": 0.0, "cost_per_test": 9.6664, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "meta", "harness": "muse-code" }, "openai/gpt-6-astra": { "accuracy": 0.909, "latency": 95.16, "stderr": 0.0, "cost_per_test": 14.1039, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 77.48, "stderr": 0.0, "cost_per_test": 7.9847, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 153.7, "stderr": 0.0, "cost_per_test": 4.7006, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": "xhigh", "provider": "grok", "harness": "grok-build" }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 166.22, "stderr": 0.0, "cost_per_test": 8.703109, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "high", "verbosity": null, "compute_effort": "high", "provider": "google", "harness": "antigravity" } }, "balance_scale": { "openai/gpt-6.1-sol": { "accuracy": 100.0, "latency": 46.54, "stderr": 0.0, "cost_per_test": 0.2038, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "anthropic/claude-opus-5-5": { "accuracy": 95.0, "latency": 143.14, "stderr": 0.0, "cost_per_test": 0.4253, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "anthropic/claude-fable-5-1": { "accuracy": 10.0, "latency": 126.42, "stderr": 0.0, "cost_per_test": 0.8025, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "meta/muse-wonder-full": { "accuracy": 10.0, "latency": 136.28, "stderr": 0.0, "cost_per_test": 0.8264, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "meta", "harness": "muse-code" }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 38.46, "stderr": 0.0, "cost_per_test": 1.2184, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 145.52, "stderr": 0.0, "cost_per_test": 0.516261, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "high", "verbosity": null, "compute_effort": "high", "provider": "google", "harness": "antigravity" }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 180.0, "stderr": 0.0, "cost_per_test": 0.511, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": "xhigh", "provider": "grok", "harness": "grok-build" } }, "lego_manual": { "openai/gpt-6-astra": { "accuracy": 15.0, "latency": 85.32, "stderr": 0.0, "cost_per_test": 1.1167, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "openai/gpt-6.1-sol": { "accuracy": 15.0, "latency": 102.76, "stderr": 0.0, "cost_per_test": 0.203, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "anthropic/claude-opus-5-5": { "accuracy": 15.0, "latency": 166.48, "stderr": 0.0, "cost_per_test": 0.467, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 180.0, "stderr": 0.0, "cost_per_test": 0.606, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": "xhigh", "provider": "grok", "harness": "grok-build" }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 180.0, "stderr": 0.0, "cost_per_test": 0.666877, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "high", "verbosity": null, "compute_effort": "high", "provider": "google", "harness": "antigravity" }, "meta/muse-wonder-full": { "accuracy": 0.0, "latency": 180.0, "stderr": 0.0, "cost_per_test": 0.7533, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "meta", "harness": "muse-code" }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 180.0, "stderr": 0.0, "cost_per_test": 1.3283, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" } }, "ninja_course": { "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 44.46, "stderr": 0.0, "cost_per_test": 0.4382, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 68.54, "stderr": 0.0, "cost_per_test": 0.2937, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "anthropic", "harness": "claude-code" }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 104.38, "stderr": 0.0, "cost_per_test": 1.2327, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 122.98, "stderr": 0.0, "cost_per_test": 0.2085, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "openai", "harness": "codex" }, "meta/muse-wonder-full": { "accuracy": 0.0, "latency": 125.64, "stderr": 0.0, "cost_per_test": 0.7915, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "max", "verbosity": null, "compute_effort": "max", "provider": "meta", "harness": "muse-code" }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 180.0, "stderr": 0.0, "cost_per_test": 0.0679, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": "xhigh", "provider": "grok", "harness": "grok-build" }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 180.0, "stderr": 0.0, "cost_per_test": 0.985995, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": true, "reasoning_effort": "high", "verbosity": null, "compute_effort": "high", "provider": "google", "harness": "antigravity" } } } }