{
    "metadata": {
        "benchmark": "Autoform-Hard",
        "slug": "autoform_hard",
        "description": "Can agents formalize graduate- and research-level mathematics in Lean 4?",
        "benchmark_id": "autoform_hard",
        "family": "autoform_hard",
        "version": "0.2",
        "updated": "2026-11-18",
        "dataset_type": "public",
        "industry": "math",
        "tasks": {
            "overall": "Overall"
        },
        "models": [
            "anthropic/claude-fable-5-2",
            "google/gemini-4-argon",
            "google/gemini-4-carbon",
            "google/gemini-4-hydrogen",
            "minimax/MiniMax-M3.1-flash",
            "openai/gpt-6.2-astra",
            "openai/gpt-6.2-sol",
            "reflection/Beam-1.0"
        ],
        "partners": [],
        "showBadge": false,
        "visible": true,
        "use_cost_per_test": true,
        "cost_label": "Cost / Task",
        "runner": "external",
        "mode": "agentic",
        "archived": false,
        "partner": false,
        "total_models": 8
    },
    "tasks": {
        "overall": {
            "openai/gpt-6.2-astra": {
                "accuracy": 87.5,
                "latency": 2968.178,
                "stderr": 4.459,
                "cost_per_test": 15.243228,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-fable-5-2": {
                "accuracy": 85.714,
                "latency": 3996.616,
                "stderr": 4.718,
                "cost_per_test": 23.214917,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6.2-sol": {
                "accuracy": 85.714,
                "latency": 4366.778,
                "stderr": 4.718,
                "cost_per_test": 8.090518,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-4-carbon": {
                "accuracy": 71.429,
                "latency": 6909.422,
                "stderr": 6.091,
                "cost_per_test": 12.377677,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 1048576,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "google/gemini-4-hydrogen": {
                "accuracy": 64.286,
                "latency": 7640.492,
                "stderr": 6.461,
                "cost_per_test": 23.253338,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 1048576,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 48.214,
                "latency": 6487.894,
                "stderr": 6.738,
                "cost_per_test": 25.736314,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 1048576,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "reflection/Beam-1.0": {
                "accuracy": 0.0,
                "latency": 3242.616,
                "stderr": 0.0,
                "cost_per_test": null,
                "temperature": 0.7,
                "top_p": 0.99,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Reflection",
                "harness": null
            },
            "minimax/MiniMax-M3.1-flash": {
                "accuracy": 0.0,
                "latency": 10631.247,
                "stderr": 0.0,
                "cost_per_test": 4.289858,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "MiniMax",
                "harness": null
            }
        }
    }
}