{ "metadata": { "benchmark": "Time Horizon Index: KSP", "slug": "time_horizon_index", "description": "Can an AI agent build and run a space program in Kerbal Space Program?", "benchmark_id": "ksp_bench", "family": "ksp_bench", "version": "1", "updated": "2026-10-07", "dataset_type": "private", "industry": "beta", "tasks": { "overall": "Overall", "mean_progress_to_date": "Progress efficiency" }, "models": [ "anthropic/claude-fable-5-1", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.5", "grok/grok-4.6", "kimi/kimi-k3", "meta/muse_spark_1_1", "openai/gpt-5.5", "openai/gpt-5.6-sol", "openai/gpt-6-astra" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "cost_label": "Recorded API cost", "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 13, "token_tasks_measured": 1 }, "tasks": { "overall": { "anthropic/claude-opus-5-5": { "accuracy": 91.333, "latency": 0.0, "stderr": 0.0, "cost_per_test": 921.761327, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null, "token_totals": { "input_tokens": 2996235143, "output_tokens": 6641667, "reasoning_tokens": null, "cache_read_tokens": 2956716165, "cache_write_tokens": 39508842 }, "token_tasks_measured": 1 }, "openai/gpt-6-astra": { "accuracy": 90.5, "latency": 0.0, "stderr": 0.0, "cost_per_test": 4206.58511, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 63.333, "latency": 0.0, "stderr": 0.0, "cost_per_test": 4655.011028, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null, "token_totals": { "input_tokens": 1154210745, "output_tokens": 2520090, "reasoning_tokens": null, "cache_read_tokens": 808050420, "cache_write_tokens": 346156269 }, "token_tasks_measured": 1 }, "google/gemini-4-argon": { "accuracy": 52.5, "latency": 0.0, "stderr": 8.856, "cost_per_test": 708.627391, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 23.833, "latency": 0.0, "stderr": 0.0, "cost_per_test": 2027.203388, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null, "token_totals": { "input_tokens": 2434036734, "output_tokens": 1371030, "reasoning_tokens": 748101, "cache_read_tokens": 2420589534, "cache_write_tokens": 13431880 }, "token_tasks_measured": 1 }, "google/gemini-3.8-flash": { "accuracy": 18.833, "latency": 0.0, "stderr": 0.0, "cost_per_test": 543.140818, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null, "token_totals": { "input_tokens": 1788590251, "output_tokens": 2749998, "reasoning_tokens": 1584982, "cache_read_tokens": 1600273736, "cache_write_tokens": 0 }, "token_tasks_measured": 1 }, "anthropic/claude-opus-5": { "accuracy": 18.833, "latency": 0.0, "stderr": 0.0, "cost_per_test": 1348.765783, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null, "token_totals": { "input_tokens": 432057935, "output_tokens": 1004615, "reasoning_tokens": null, "cache_read_tokens": 239427694, "cache_write_tokens": 192628285 }, "token_tasks_measured": 1 }, "anthropic/claude-opus-4-8": { "accuracy": 11.833, "latency": 0.0, "stderr": 0.0, "cost_per_test": 1499.319504, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 10.5, "latency": 0.0, "stderr": 5.38, "cost_per_test": 216.426849, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null, "token_totals": { "input_tokens": 374582733, "output_tokens": 1078246, "reasoning_tokens": 782198, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 1 }, "openai/gpt-5.5": { "accuracy": 8.333, "latency": 0.0, "stderr": 0.0, "cost_per_test": 516.360641, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 6.333, "latency": 0.0, "stderr": 0.0, "cost_per_test": 344.035038, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 5.833, "latency": 0.0, "stderr": 0.0, "cost_per_test": 225.11666, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null, "token_totals": { "input_tokens": 288701086, "output_tokens": 1096788, "reasoning_tokens": 652001, "cache_read_tokens": 239244160, "cache_write_tokens": 0 }, "token_tasks_measured": 1 }, "meta/muse_spark_1_1": { "accuracy": 1.667, "latency": 0.0, "stderr": 0.0, "cost_per_test": 144.921195, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null } }, "mean_progress_to_date": { "openai/gpt-6-astra": { "accuracy": 70.112, "latency": 0.0, "stderr": 0.0, "cost_per_test": 4206.58511, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 69.825, "latency": 0.0, "stderr": 0.0, "cost_per_test": 921.761327, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null, "token_totals": { "input_tokens": 2996235143, "output_tokens": 6641667, "reasoning_tokens": null, "cache_read_tokens": 2956716165, "cache_write_tokens": 39508842 }, "token_tasks_measured": 1 }, "google/gemini-4-argon": { "accuracy": 40.34, "latency": 0.0, "stderr": 0.0, "cost_per_test": 708.627391, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 33.384, "latency": 0.0, "stderr": 0.0, "cost_per_test": 4655.011028, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null, "token_totals": { "input_tokens": 1154210745, "output_tokens": 2520090, "reasoning_tokens": null, "cache_read_tokens": 808050420, "cache_write_tokens": 346156269 }, "token_tasks_measured": 1 }, "anthropic/claude-opus-5": { "accuracy": 17.527, "latency": 0.0, "stderr": 0.0, "cost_per_test": 1348.765783, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null, "token_totals": { "input_tokens": 432057935, "output_tokens": 1004615, "reasoning_tokens": null, "cache_read_tokens": 239427694, "cache_write_tokens": 192628285 }, "token_tasks_measured": 1 }, "google/gemini-3.8-flash": { "accuracy": 14.5, "latency": 0.0, "stderr": 0.0, "cost_per_test": 543.140818, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null, "token_totals": { "input_tokens": 1788590251, "output_tokens": 2749998, "reasoning_tokens": 1584982, "cache_read_tokens": 1600273736, "cache_write_tokens": 0 }, "token_tasks_measured": 1 }, "openai/gpt-5.6-sol": { "accuracy": 13.662, "latency": 0.0, "stderr": 0.0, "cost_per_test": 2027.203388, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null, "token_totals": { "input_tokens": 2434036734, "output_tokens": 1371030, "reasoning_tokens": 748101, "cache_read_tokens": 2420589534, "cache_write_tokens": 13431880 }, "token_tasks_measured": 1 }, "kimi/kimi-k3": { "accuracy": 9.258, "latency": 0.0, "stderr": 0.0, "cost_per_test": 216.426849, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null, "token_totals": { "input_tokens": 374582733, "output_tokens": 1078246, "reasoning_tokens": 782198, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 1 }, "anthropic/claude-opus-4-8": { "accuracy": 7.739, "latency": 0.0, "stderr": 0.0, "cost_per_test": 1499.319504, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 6.032, "latency": 0.0, "stderr": 0.0, "cost_per_test": 516.360641, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 5.939, "latency": 0.0, "stderr": 0.0, "cost_per_test": 344.035038, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 4.88, "latency": 0.0, "stderr": 0.0, "cost_per_test": 225.11666, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null, "token_totals": { "input_tokens": 288701086, "output_tokens": 1096788, "reasoning_tokens": 652001, "cache_read_tokens": 239244160, "cache_write_tokens": 0 }, "token_tasks_measured": 1 }, "meta/muse_spark_1_1": { "accuracy": 0.696, "latency": 0.0, "stderr": 0.0, "cost_per_test": 144.921195, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null } } } }