{
    "metadata": {
        "benchmark": "Time Horizon Index: KSP",
        "slug": "time_horizon_index",
        "description": "Can an AI agent build and run a space program in Kerbal Space Program?",
        "benchmark_id": "ksp_bench",
        "family": "ksp_bench",
        "version": "1",
        "updated": "2026-10-07",
        "dataset_type": "private",
        "industry": "beta",
        "tasks": {
            "overall": "Overall",
            "mean_progress_to_date": "Progress efficiency"
        },
        "models": [
            "anthropic/claude-fable-5-1",
            "anthropic/claude-opus-4-8",
            "anthropic/claude-opus-5",
            "anthropic/claude-opus-5-5",
            "google/gemini-3.8-flash",
            "google/gemini-4-argon",
            "grok/grok-4.5",
            "grok/grok-4.6",
            "kimi/kimi-k3",
            "meta/muse_spark_1_1",
            "openai/gpt-5.5",
            "openai/gpt-5.6-sol",
            "openai/gpt-6-astra"
        ],
        "partners": [],
        "showBadge": false,
        "visible": true,
        "use_cost_per_test": true,
        "cost_label": "Recorded API cost",
        "runner": "external",
        "mode": "agentic",
        "archived": false,
        "partner": false,
        "total_models": 13
    },
    "tasks": {
        "overall": {
            "anthropic/claude-opus-5-5": {
                "accuracy": 91.333,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 1303.37376,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6-astra": {
                "accuracy": 90.5,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 4206.58511,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-fable-5-1": {
                "accuracy": 63.333,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 4655.011028,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 52.5,
                "latency": 0.0,
                "stderr": 8.856,
                "cost_per_test": 708.627391,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-5.6-sol": {
                "accuracy": 23.833,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 2027.203388,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 18.833,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 543.140818,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-opus-5": {
                "accuracy": 18.833,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 1348.765783,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-opus-4-8": {
                "accuracy": 11.833,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 1499.319504,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "kimi/kimi-k3": {
                "accuracy": 10.5,
                "latency": 0.0,
                "stderr": 5.38,
                "cost_per_test": 216.426849,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "openai/gpt-5.5": {
                "accuracy": 8.333,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 516.360641,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4.5": {
                "accuracy": 6.333,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 344.035038,
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 5.833,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 225.11666,
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "meta/muse_spark_1_1": {
                "accuracy": 1.667,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 144.921195,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 256000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            }
        },
        "mean_progress_to_date": {
            "openai/gpt-6-astra": {
                "accuracy": 70.112,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 4206.58511,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-opus-5-5": {
                "accuracy": 69.825,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 1303.37376,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 40.34,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 708.627391,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-fable-5-1": {
                "accuracy": 33.384,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 4655.011028,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-opus-5": {
                "accuracy": 17.527,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 1348.765783,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 14.5,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 543.140818,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-5.6-sol": {
                "accuracy": 13.662,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 2027.203388,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "kimi/kimi-k3": {
                "accuracy": 9.258,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 216.426849,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "anthropic/claude-opus-4-8": {
                "accuracy": 7.739,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 1499.319504,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-5.5": {
                "accuracy": 6.032,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 516.360641,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4.5": {
                "accuracy": 5.939,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 344.035038,
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 4.88,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 225.11666,
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "meta/muse_spark_1_1": {
                "accuracy": 0.696,
                "latency": 0.0,
                "stderr": 0.0,
                "cost_per_test": 144.921195,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 256000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            }
        }
    }
}