{
    "metadata": {
        "benchmark": "Vals Science Index",
        "slug": "science_index",
        "description": "A single measure of AI's ability to do science \u2014 agentic model performance on biology data investigation, mechanism discovery, and research workflows, weighted equally.",
        "benchmark_id": "science_index",
        "family": "science_index",
        "version": "1.0",
        "updated": "2026-11-19",
        "dataset_type": "private",
        "industry": "index",
        "tasks": {
            "overall": "Overall",
            "biomysterybench": "BioMysteryBench",
            "mysterymechanism": "MysteryMechanism",
            "terminal_bench_science": "Terminal-Bench-Science"
        },
        "models": [
            "anthropic/claude-fable-5-2",
            "anthropic/claude-haiku-5-5",
            "anthropic/claude-opus-5",
            "anthropic/claude-opus-5-5",
            "anthropic/claude-sonnet-5-5",
            "deepseek/deepseek-v4.1-flash",
            "fireworks/ember-1",
            "google/cobalt-c-high-1p00",
            "google/gemini-3.1-pro-preview",
            "google/gemini-3.8-flash",
            "google/gemini-4-argon",
            "grok/grok-4.6",
            "grok/grok-4.7",
            "kimi/kimi-k3.1-pro",
            "meta/muse-wonder",
            "meta/muse-wonder-full",
            "openai/gpt-5.6-luna",
            "openai/gpt-5.6-sol",
            "openai/gpt-6-astra",
            "openai/gpt-6-luna",
            "openai/gpt-6-sol",
            "openai/gpt-6.1-sol",
            "openai/gpt-6.2-astra",
            "openai/gpt-6.2-sol",
            "reflection/Beam-1.0",
            "stepfun/step-5-preview",
            "thinkingmachines/inkling-2-small",
            "thinkingmachines/inkling-mini-1117",
            "xiaomi/mimo-v2.6-flash"
        ],
        "partners": [],
        "showBadge": false,
        "visible": true,
        "use_cost_per_test": true,
        "runner": "external",
        "mode": "agentic",
        "archived": false,
        "partner": false,
        "total_models": 29
    },
    "tasks": {
        "overall": {
            "openai/gpt-6.2-astra": {
                "accuracy": 68.441,
                "latency": 1334.968,
                "stderr": 2.134,
                "cost_per_test": 4.763591,
                "avg_input_tokens": 3433887.1,
                "avg_output_tokens": 48378.3,
                "avg_reasoning_tokens": 28667.8,
                "avg_cache_read_tokens": 3343737.5,
                "avg_cache_write_tokens": 89419.6,
                "token_totals": {
                    "input_tokens": 819320564,
                    "output_tokens": 14630371,
                    "reasoning_tokens": 9017971,
                    "cache_read_tokens": 792832966,
                    "cache_write_tokens": 26162481
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-6-astra": {
                "accuracy": 65.09,
                "latency": 2436.537,
                "stderr": 2.262,
                "cost_per_test": 8.027277,
                "avg_input_tokens": 2128848.1,
                "avg_output_tokens": 65321.0,
                "avg_reasoning_tokens": 49087.2,
                "avg_cache_read_tokens": 2012838.6,
                "token_totals": {
                    "input_tokens": 479860525,
                    "output_tokens": 17265188,
                    "reasoning_tokens": 12943157,
                    "cache_read_tokens": 449914908,
                    "cache_write_tokens": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-fable-5-2": {
                "accuracy": 63.74,
                "latency": 2080.932,
                "stderr": 2.3,
                "cost_per_test": 8.790492,
                "avg_input_tokens": 15224724.0,
                "avg_output_tokens": 94052.8,
                "avg_reasoning_tokens": 58229.4,
                "avg_cache_read_tokens": 15024205.5,
                "avg_cache_write_tokens": 200375.6,
                "token_totals": {
                    "input_tokens": 3261496945,
                    "output_tokens": 24413018,
                    "reasoning_tokens": 15460287,
                    "cache_read_tokens": 3212179844,
                    "cache_write_tokens": 49280492
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "google/cobalt-c-high-1p00": {
                "accuracy": 62.634,
                "latency": 2452.307,
                "stderr": 2.327,
                "cost_per_test": 8.259574,
                "avg_input_tokens": 15942757.4,
                "avg_output_tokens": 220508.3,
                "avg_reasoning_tokens": 108550.4,
                "avg_cache_read_tokens": 15040117.4,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 4592231309,
                    "output_tokens": 88015641,
                    "reasoning_tokens": 44928469,
                    "cache_read_tokens": 4302901233,
                    "cache_write_tokens": 0
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 1048576,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-6.1-sol": {
                "accuracy": 59.628,
                "latency": 1990.571,
                "stderr": 2.307,
                "cost_per_test": 1.364659,
                "avg_input_tokens": 1839269.8,
                "avg_output_tokens": 68765.2,
                "avg_reasoning_tokens": 53285.9,
                "avg_cache_read_tokens": 1719227.5,
                "avg_cache_write_tokens": 119370.2,
                "token_totals": {
                    "input_tokens": 419270976,
                    "output_tokens": 17935834,
                    "reasoning_tokens": 13763480,
                    "cache_read_tokens": 388775477,
                    "cache_write_tokens": 30189914
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-sonnet-5-5": {
                "accuracy": 59.118,
                "latency": 4158.815,
                "stderr": 2.305,
                "cost_per_test": 8.499606,
                "avg_input_tokens": 47739135.0,
                "avg_output_tokens": 281130.5,
                "avg_reasoning_tokens": 198318.2,
                "avg_cache_read_tokens": 47358254.9,
                "avg_cache_write_tokens": 380571.6,
                "token_totals": {
                    "input_tokens": 10935628677,
                    "output_tokens": 91624203,
                    "reasoning_tokens": 67740975,
                    "cache_read_tokens": 10815790916,
                    "cache_write_tokens": 119754753
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-opus-5-5": {
                "accuracy": 58.651,
                "latency": 3591.274,
                "stderr": 2.318,
                "cost_per_test": 8.710319,
                "avg_input_tokens": 11029944.8,
                "avg_output_tokens": 150132.5,
                "avg_reasoning_tokens": 107140.1,
                "avg_cache_read_tokens": 10321029.3,
                "avg_cache_write_tokens": 708727.1,
                "token_totals": {
                    "input_tokens": 2847006558,
                    "output_tokens": 52917807,
                    "reasoning_tokens": 39539150,
                    "cache_read_tokens": 2669242789,
                    "cache_write_tokens": 177708111
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6.2-sol": {
                "accuracy": 57.762,
                "latency": 1140.079,
                "stderr": 2.264,
                "cost_per_test": 1.119045,
                "avg_input_tokens": 3161858.5,
                "avg_output_tokens": 47721.0,
                "avg_reasoning_tokens": 26898.8,
                "avg_cache_read_tokens": 3062722.5,
                "avg_cache_write_tokens": 98430.1,
                "token_totals": {
                    "input_tokens": 755525041,
                    "output_tokens": 14979406,
                    "reasoning_tokens": 8808947,
                    "cache_read_tokens": 725503672,
                    "cache_write_tokens": 29704339
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 55.359,
                "latency": 3420.286,
                "stderr": 2.288,
                "cost_per_test": 18.402067,
                "avg_input_tokens": 12671021.4,
                "avg_output_tokens": 196916.0,
                "avg_reasoning_tokens": 107584.4,
                "avg_cache_read_tokens": 9531668.2,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 3319898414,
                    "output_tokens": 59740582,
                    "reasoning_tokens": 32538408,
                    "cache_read_tokens": 2578283096,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "meta/muse-wonder-full": {
                "accuracy": 48.537,
                "latency": 5662.568,
                "stderr": 2.225,
                "cost_per_test": 10.150213,
                "avg_input_tokens": 21381459.4,
                "avg_output_tokens": 136611.4,
                "avg_reasoning_tokens": 90148.9,
                "avg_cache_read_tokens": 20543352.6,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 5132642295,
                    "output_tokens": 45144071,
                    "reasoning_tokens": 29757091,
                    "cache_read_tokens": 4891569963,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "anthropic/claude-opus-5": {
                "accuracy": 47.93,
                "latency": 3705.653,
                "stderr": 2.199,
                "cost_per_test": 13.038951,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6-sol": {
                "accuracy": 44.998,
                "latency": 1876.106,
                "stderr": 2.258,
                "cost_per_test": 2.314922,
                "avg_input_tokens": 5859531.3,
                "avg_output_tokens": 61916.1,
                "avg_reasoning_tokens": 39405.9,
                "avg_cache_read_tokens": 5753148.1,
                "avg_cache_write_tokens": 105496.6,
                "token_totals": {
                    "input_tokens": 1309596348,
                    "output_tokens": 18131437,
                    "reasoning_tokens": 12283114,
                    "cache_read_tokens": 1280231223,
                    "cache_write_tokens": 29006459
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "kimi/kimi-k3.1-pro": {
                "accuracy": 44.37,
                "latency": 1382.066,
                "stderr": 2.118,
                "cost_per_test": 3.329791,
                "avg_input_tokens": 11754337.6,
                "avg_output_tokens": 71898.9,
                "avg_reasoning_tokens": 44692.3,
                "avg_cache_read_tokens": 11609929.6,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 2605725079,
                    "output_tokens": 21578560,
                    "reasoning_tokens": 14319337,
                    "cache_read_tokens": 2565318485,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "openai/gpt-5.6-sol": {
                "accuracy": 41.481,
                "latency": 2612.344,
                "stderr": 1.934,
                "cost_per_test": 3.403486,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "meta/muse-wonder": {
                "accuracy": 39.124,
                "latency": 2587.679,
                "stderr": 1.964,
                "cost_per_test": 8.742674,
                "avg_input_tokens": 24760143.2,
                "avg_output_tokens": 137061.0,
                "avg_reasoning_tokens": 89605.8,
                "avg_cache_read_tokens": 24476475.7,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 5899322456,
                    "output_tokens": 45376332,
                    "reasoning_tokens": 29190884,
                    "cache_read_tokens": 5810889987,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "anthropic/claude-haiku-5-5": {
                "accuracy": 38.502,
                "latency": 2398.32,
                "stderr": 1.671,
                "cost_per_test": 1.859159,
                "avg_input_tokens": 25901386.0,
                "avg_output_tokens": 248671.8,
                "avg_reasoning_tokens": 194764.8,
                "avg_cache_read_tokens": 25581293.2,
                "avg_cache_write_tokens": 319900.1,
                "token_totals": {
                    "input_tokens": 6116009622,
                    "output_tokens": 84339072,
                    "reasoning_tokens": 67306753,
                    "cache_read_tokens": 6010191105,
                    "cache_write_tokens": 105764540
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 38.094,
                "latency": 2409.528,
                "stderr": 1.643,
                "cost_per_test": 2.450732,
                "avg_input_tokens": 3102681.5,
                "avg_output_tokens": 85489.1,
                "avg_reasoning_tokens": 53741.9,
                "avg_cache_read_tokens": 2845043.6,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 784994822,
                    "output_tokens": 35220835,
                    "reasoning_tokens": 23972346,
                    "cache_read_tokens": 695373781,
                    "cache_write_tokens": 0
                },
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 35.76,
                "latency": 1320.142,
                "stderr": 1.657,
                "cost_per_test": 2.975983,
                "avg_input_tokens": 9160328.3,
                "avg_output_tokens": 94594.4,
                "avg_reasoning_tokens": 50072.7,
                "avg_cache_read_tokens": 8498088.2,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 2571424774,
                    "output_tokens": 32899229,
                    "reasoning_tokens": 17671298,
                    "cache_read_tokens": 2360671595,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "grok/grok-4.7": {
                "accuracy": 34.828,
                "latency": 2434.191,
                "stderr": 1.611,
                "cost_per_test": 7.123405,
                "avg_input_tokens": 6973663.7,
                "avg_output_tokens": 97507.0,
                "avg_reasoning_tokens": 54047.2,
                "avg_cache_read_tokens": 6350166.2,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 1756429074,
                    "output_tokens": 33549403,
                    "reasoning_tokens": 18014691,
                    "cache_read_tokens": 1558762453,
                    "cache_write_tokens": 0
                },
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "thinkingmachines/inkling-2-small": {
                "accuracy": 32.694,
                "latency": 3449.38,
                "stderr": 1.256,
                "cost_per_test": 1.991855,
                "avg_input_tokens": 70528606.5,
                "avg_output_tokens": 208106.7,
                "avg_cache_read_tokens": 70229569.5,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 15165861610,
                    "output_tokens": 57919044,
                    "reasoning_tokens": null,
                    "cache_read_tokens": 15085673643,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "fireworks/ember-1": {
                "accuracy": 31.758,
                "latency": 4659.396,
                "stderr": 1.663,
                "cost_per_test": 10.987134,
                "avg_input_tokens": 27227565.1,
                "avg_output_tokens": 122075.9,
                "avg_reasoning_tokens": 84610.8,
                "avg_cache_read_tokens": 26861740.4,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 5841634319,
                    "output_tokens": 35225666,
                    "reasoning_tokens": 25397188,
                    "cache_read_tokens": 5752309510,
                    "cache_write_tokens": 0
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Fireworks AI",
                "harness": null
            },
            "xiaomi/mimo-v2.6-flash": {
                "accuracy": 31.722,
                "latency": 6047.487,
                "stderr": 1.272,
                "cost_per_test": 0.117321,
                "avg_input_tokens": 11978208.2,
                "avg_output_tokens": 157495.6,
                "avg_reasoning_tokens": 119701.7,
                "avg_cache_read_tokens": 11688973.3,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 2838987541,
                    "output_tokens": 52127714,
                    "reasoning_tokens": 40551870,
                    "cache_read_tokens": 2752730944,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "deepseek/deepseek-v4.1-flash": {
                "accuracy": 29.65,
                "latency": 3998.656,
                "stderr": 0.988,
                "cost_per_test": 0.290142,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 384000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "stepfun/step-5-preview": {
                "accuracy": 29.577,
                "latency": 4017.643,
                "stderr": 1.104,
                "cost_per_test": 1.780912,
                "avg_input_tokens": 21215985.1,
                "avg_output_tokens": 170536.4,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 20942653.9,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 4827403507,
                    "output_tokens": 56528893,
                    "reasoning_tokens": 0,
                    "cache_read_tokens": 4742501376,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Stepfun",
                "harness": null
            },
            "openai/gpt-6-luna": {
                "accuracy": 28.379,
                "latency": 1814.828,
                "stderr": 1.481,
                "cost_per_test": 0.106685,
                "avg_input_tokens": 4354905.1,
                "avg_output_tokens": 76098.2,
                "avg_reasoning_tokens": 58346.2,
                "avg_cache_read_tokens": 4235499.6,
                "avg_cache_write_tokens": 118345.5,
                "token_totals": {
                    "input_tokens": 994942775,
                    "output_tokens": 22486545,
                    "reasoning_tokens": 17799973,
                    "cache_read_tokens": 961803372,
                    "cache_write_tokens": 32735080
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-3.1-pro-preview": {
                "accuracy": 27.383,
                "latency": 1083.463,
                "stderr": 1.419,
                "cost_per_test": 2.561873,
                "avg_input_tokens": 4761406.8,
                "avg_output_tokens": 39111.0,
                "avg_reasoning_tokens": 20031.9,
                "avg_cache_read_tokens": 4350920.3,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 1195483910,
                    "output_tokens": 14275349,
                    "reasoning_tokens": 6920343,
                    "cache_read_tokens": 1074011989,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-5.6-luna": {
                "accuracy": 25.298,
                "latency": 3000.259,
                "stderr": 0.825,
                "cost_per_test": 0.259854,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "thinkingmachines/inkling-mini-1117": {
                "accuracy": 23.584,
                "latency": 2728.755,
                "stderr": 1.1,
                "cost_per_test": 1.658497,
                "avg_input_tokens": 51333399.4,
                "avg_output_tokens": 176554.1,
                "avg_cache_read_tokens": 51082031.9,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 11292518686,
                    "output_tokens": 54670922,
                    "reasoning_tokens": null,
                    "cache_read_tokens": 11217566720,
                    "cache_write_tokens": 0
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "reflection/Beam-1.0": {
                "accuracy": 22.031,
                "latency": 1747.731,
                "stderr": 1.175,
                "cost_per_test": null,
                "avg_input_tokens": 11004312.3,
                "avg_output_tokens": 90657.7,
                "avg_reasoning_tokens": 69036.0,
                "avg_cache_read_tokens": 10807205.0,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 2577555254,
                    "output_tokens": 28016811,
                    "reasoning_tokens": 20730785,
                    "cache_read_tokens": 2518905088,
                    "cache_write_tokens": 0
                },
                "temperature": 0.7,
                "top_p": 0.99,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Reflection",
                "harness": null
            }
        },
        "biomysterybench": {
            "anthropic/claude-fable-5-2": {
                "accuracy": 83.704,
                "latency": 248.169,
                "stderr": 0.741,
                "cost_per_test": 1.681583,
                "token_totals": {
                    "input_tokens": 81792662,
                    "output_tokens": 1977797,
                    "reasoning_tokens": 732302,
                    "cache_read_tokens": 75949519,
                    "cache_write_tokens": 5836588
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 18.4,
                    "avg_query_seconds": 13.5,
                    "median_query_seconds": 8.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-sonnet-5-5": {
                "accuracy": 81.111,
                "latency": 1086.061,
                "stderr": 0.642,
                "cost_per_test": 2.945242,
                "token_totals": {
                    "input_tokens": 732589448,
                    "output_tokens": 14342557,
                    "reasoning_tokens": 7945289,
                    "cache_read_tokens": 712471480,
                    "cache_write_tokens": 20094500
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 62.3,
                    "avg_query_seconds": 17.5,
                    "median_query_seconds": 6.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6.2-astra": {
                "accuracy": 80.741,
                "latency": 509.913,
                "stderr": 0.37,
                "cost_per_test": 1.488087,
                "token_totals": {
                    "input_tokens": 50839131,
                    "output_tokens": 1793226,
                    "reasoning_tokens": 563822,
                    "cache_read_tokens": 46900725,
                    "cache_write_tokens": 3851921
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 16.5,
                    "avg_query_seconds": 30.9,
                    "median_query_seconds": 14.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/cobalt-c-high-1p00": {
                "accuracy": 80.0,
                "latency": 1103.661,
                "stderr": 1.925,
                "cost_per_test": 4.40725,
                "token_totals": {
                    "input_tokens": 519931856,
                    "output_tokens": 9950549,
                    "reasoning_tokens": 5599702,
                    "cache_read_tokens": 459355927,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 41.5,
                    "avg_query_seconds": 26.6,
                    "median_query_seconds": 16.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 1048576,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-6.1-sol": {
                "accuracy": 79.63,
                "latency": 530.7,
                "stderr": 0.741,
                "cost_per_test": 0.372147,
                "token_totals": {
                    "input_tokens": 49187181,
                    "output_tokens": 1921230,
                    "reasoning_tokens": 1213195,
                    "cache_read_tokens": 45054185,
                    "cache_write_tokens": 4050774
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 17.4,
                    "avg_query_seconds": 30.5,
                    "median_query_seconds": 16.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-6-astra": {
                "accuracy": 79.259,
                "latency": 286.147,
                "stderr": 0.98,
                "cost_per_test": 1.719326,
                "token_totals": {
                    "input_tokens": 39863504,
                    "output_tokens": 1463159,
                    "reasoning_tokens": 815272,
                    "cache_read_tokens": 35228179,
                    "cache_write_tokens": null
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 16.6,
                    "avg_query_seconds": 17.2,
                    "median_query_seconds": 11.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-opus-5-5": {
                "accuracy": 79.259,
                "latency": 739.252,
                "stderr": 0.98,
                "cost_per_test": 2.947297,
                "token_totals": {
                    "input_tokens": 359126168,
                    "output_tokens": 6342038,
                    "reasoning_tokens": 4159050,
                    "cache_read_tokens": 348450323,
                    "cache_write_tokens": 10652845
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 43.5,
                    "avg_query_seconds": 17.0,
                    "median_query_seconds": 8.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-opus-5": {
                "accuracy": 79.259,
                "latency": 2195.964,
                "stderr": 2.062,
                "cost_per_test": 3.292774,
                "turn_metrics": {
                    "tasks_measured": 88,
                    "avg_turns": 33.4,
                    "avg_query_seconds": 25.1,
                    "median_query_seconds": 15.2,
                    "avg_ttft_seconds": 17.7,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6.2-sol": {
                "accuracy": 77.778,
                "latency": 422.4,
                "stderr": 0.642,
                "cost_per_test": 0.338256,
                "token_totals": {
                    "input_tokens": 39128444,
                    "output_tokens": 1740145,
                    "reasoning_tokens": 821634,
                    "cache_read_tokens": 35307819,
                    "cache_write_tokens": 3739101
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 14.8,
                    "avg_query_seconds": 28.6,
                    "median_query_seconds": 18.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 76.296,
                "latency": 2396.816,
                "stderr": 0.37,
                "cost_per_test": 5.707134,
                "token_totals": {
                    "input_tokens": 187578219,
                    "output_tokens": 16476596,
                    "reasoning_tokens": 11274521,
                    "cache_read_tokens": 149000710,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 23.7,
                    "avg_query_seconds": 101.1,
                    "median_query_seconds": 25.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "kimi/kimi-k3.1-pro": {
                "accuracy": 75.556,
                "latency": 311.42,
                "stderr": 0.642,
                "cost_per_test": 0.789777,
                "token_totals": {
                    "input_tokens": 140611110,
                    "output_tokens": 3102396,
                    "reasoning_tokens": 1963256,
                    "cache_read_tokens": 133981269,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 27.7,
                    "avg_query_seconds": 11.2,
                    "median_query_seconds": 6.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "openai/gpt-6-sol": {
                "accuracy": 74.815,
                "latency": 324.465,
                "stderr": 2.429,
                "cost_per_test": 0.662358,
                "token_totals": {
                    "input_tokens": 129632435,
                    "output_tokens": 2217483,
                    "reasoning_tokens": 1401831,
                    "cache_read_tokens": 124856548,
                    "cache_write_tokens": 4671549
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 28.3,
                    "avg_query_seconds": 11.5,
                    "median_query_seconds": 6.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "meta/muse-wonder-full": {
                "accuracy": 73.333,
                "latency": 1574.165,
                "stderr": 0.642,
                "cost_per_test": 3.473154,
                "token_totals": {
                    "input_tokens": 712304421,
                    "output_tokens": 5992307,
                    "reasoning_tokens": 3536726,
                    "cache_read_tokens": 696010395,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 78.1,
                    "avg_query_seconds": 20.2,
                    "median_query_seconds": 8.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 72.222,
                "latency": 858.019,
                "stderr": 0.0,
                "cost_per_test": 1.226606,
                "token_totals": {
                    "input_tokens": 104073191,
                    "output_tokens": 4208045,
                    "reasoning_tokens": 2899195,
                    "cache_read_tokens": 82000085,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 30.9,
                    "avg_query_seconds": 27.8,
                    "median_query_seconds": 10.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "openai/gpt-5.6-sol": {
                "accuracy": 71.111,
                "latency": 1098.261,
                "stderr": 0.642,
                "cost_per_test": 1.925036,
                "turn_metrics": {
                    "tasks_measured": 89,
                    "avg_turns": 22.4,
                    "avg_query_seconds": 27.2,
                    "median_query_seconds": 12.0,
                    "avg_ttft_seconds": 7.7,
                    "output_time_split": {
                        "reasoning_pct": 12.1,
                        "content_pct": 87.9,
                        "tool_call_pct": 0.0
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-haiku-5-5": {
                "accuracy": 70.37,
                "latency": 752.457,
                "stderr": 1.335,
                "cost_per_test": 0.512756,
                "token_totals": {
                    "input_tokens": 241182764,
                    "output_tokens": 16695865,
                    "reasoning_tokens": 14393004,
                    "cache_read_tokens": 217680976,
                    "cache_write_tokens": 23494403
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 20.5,
                    "avg_query_seconds": 36.7,
                    "median_query_seconds": 15.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "thinkingmachines/inkling-2-small": {
                "accuracy": 70.0,
                "latency": 468.444,
                "stderr": 1.283,
                "cost_per_test": 0.268628,
                "avg_input_tokens": 5419257.4,
                "avg_output_tokens": 71904.0,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 5297982.7,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 484395280,
                    "output_tokens": 6424541,
                    "reasoning_tokens": 0,
                    "cache_read_tokens": 473559211,
                    "cache_write_tokens": 0
                },
                "turn_metrics": {
                    "tasks_measured": 89,
                    "avg_turns": 59.2,
                    "avg_query_seconds": 7.9,
                    "median_query_seconds": 3.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "grok/grok-4.7": {
                "accuracy": 69.259,
                "latency": 696.818,
                "stderr": 1.335,
                "cost_per_test": 2.487742,
                "token_totals": {
                    "input_tokens": 254511912,
                    "output_tokens": 3699044,
                    "reasoning_tokens": 2064454,
                    "cache_read_tokens": 215793109,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 44.4,
                    "avg_query_seconds": 15.7,
                    "median_query_seconds": 10.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "xiaomi/mimo-v2.6-flash": {
                "accuracy": 69.259,
                "latency": 1707.953,
                "stderr": 0.98,
                "cost_per_test": 0.050119,
                "token_totals": {
                    "input_tokens": 363783190,
                    "output_tokens": 6591713,
                    "reasoning_tokens": 4865655,
                    "cache_read_tokens": 351782592,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 47.5,
                    "avg_query_seconds": 35.9,
                    "median_query_seconds": 16.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "deepseek/deepseek-v4.1-flash": {
                "accuracy": 67.778,
                "latency": 890.067,
                "stderr": 1.111,
                "cost_per_test": 0.233183,
                "token_totals": {
                    "input_tokens": 268679692,
                    "output_tokens": 14743987,
                    "reasoning_tokens": 13033344,
                    "cache_read_tokens": 262450176,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 50.7,
                    "avg_query_seconds": 17.6,
                    "median_query_seconds": 10.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 384000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "stepfun/step-5-preview": {
                "accuracy": 67.407,
                "latency": 1294.751,
                "stderr": 0.37,
                "cost_per_test": 0.494625,
                "token_totals": {
                    "input_tokens": 270017689,
                    "output_tokens": 7012589,
                    "reasoning_tokens": 0,
                    "cache_read_tokens": 257300480,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 47.8,
                    "avg_query_seconds": 27.1,
                    "median_query_seconds": 10.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Stepfun",
                "harness": null
            },
            "fireworks/ember-1": {
                "accuracy": 67.037,
                "latency": 566.167,
                "stderr": 3.032,
                "cost_per_test": 0.907066,
                "token_totals": {
                    "input_tokens": 127490411,
                    "output_tokens": 1806362,
                    "reasoning_tokens": 999895,
                    "cache_read_tokens": 121455841,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 36.4,
                    "avg_query_seconds": 15.6,
                    "median_query_seconds": 9.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Fireworks AI",
                "harness": null
            },
            "meta/muse-wonder": {
                "accuracy": 67.037,
                "latency": 1081.905,
                "stderr": 0.37,
                "cost_per_test": 3.228928,
                "token_totals": {
                    "input_tokens": 656406298,
                    "output_tokens": 4876338,
                    "reasoning_tokens": 2954517,
                    "cache_read_tokens": 638118607,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 78.0,
                    "avg_query_seconds": 13.9,
                    "median_query_seconds": 5.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 62.222,
                "latency": 326.755,
                "stderr": 1.697,
                "cost_per_test": 1.5539,
                "avg_input_tokens": 4436113.1,
                "avg_output_tokens": 44052.3,
                "avg_reasoning_tokens": 22053.7,
                "avg_cache_read_tokens": 4019272.0,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 391907840,
                    "output_tokens": 3890918,
                    "reasoning_tokens": 1948190,
                    "cache_read_tokens": 355089274,
                    "cache_write_tokens": 0
                },
                "turn_metrics": {
                    "tasks_measured": 88,
                    "avg_turns": 68.5,
                    "avg_query_seconds": 4.8,
                    "median_query_seconds": 3.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-6-luna": {
                "accuracy": 61.481,
                "latency": 523.805,
                "stderr": 2.593,
                "cost_per_test": 0.053236,
                "token_totals": {
                    "input_tokens": 157508494,
                    "output_tokens": 4482770,
                    "reasoning_tokens": 3258485,
                    "cache_read_tokens": 149567191,
                    "cache_write_tokens": 7787108
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 32.8,
                    "avg_query_seconds": 16.0,
                    "median_query_seconds": 6.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-5.6-luna": {
                "accuracy": 61.481,
                "latency": 1111.274,
                "stderr": 0.741,
                "cost_per_test": 0.088859,
                "turn_metrics": {
                    "tasks_measured": 89,
                    "avg_turns": 32.8,
                    "avg_query_seconds": 12.1,
                    "median_query_seconds": 7.4,
                    "avg_ttft_seconds": 4.2,
                    "output_time_split": {
                        "reasoning_pct": 35.7,
                        "content_pct": 64.3,
                        "tool_call_pct": 0.0
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-3.1-pro-preview": {
                "accuracy": 60.0,
                "latency": 251.867,
                "stderr": 2.94,
                "cost_per_test": 0.713378,
                "avg_input_tokens": 835923.1,
                "avg_output_tokens": 23939.3,
                "avg_reasoning_tokens": 14775.9,
                "avg_cache_read_tokens": 701510.9,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 74631111,
                    "output_tokens": 2138194,
                    "reasoning_tokens": 1319799,
                    "cache_read_tokens": 62626251,
                    "cache_write_tokens": 0
                },
                "turn_metrics": {
                    "tasks_measured": 89,
                    "avg_turns": 24.8,
                    "avg_query_seconds": 10.2,
                    "median_query_seconds": 7.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "thinkingmachines/inkling-mini-1117": {
                "accuracy": 47.778,
                "latency": 550.238,
                "stderr": 1.697,
                "cost_per_test": 0.259374,
                "avg_input_tokens": 6811902.6,
                "avg_output_tokens": 78071.1,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 6680067.8,
                "avg_cache_write_tokens": 0.0,
                "token_totals": {
                    "input_tokens": 608942926,
                    "output_tokens": 6977422,
                    "reasoning_tokens": 0,
                    "cache_read_tokens": 597161728,
                    "cache_write_tokens": 0
                },
                "turn_metrics": {
                    "tasks_measured": 89,
                    "avg_turns": 76.4,
                    "avg_query_seconds": 7.2,
                    "median_query_seconds": 3.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "reflection/Beam-1.0": {
                "accuracy": 42.593,
                "latency": 440.046,
                "stderr": 1.614,
                "cost_per_test": null,
                "token_totals": {
                    "input_tokens": 548005346,
                    "output_tokens": 15353470,
                    "reasoning_tokens": 13675718,
                    "cache_read_tokens": 524174592,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 90,
                "turn_metrics": {
                    "tasks_measured": 90,
                    "avg_turns": 60.3,
                    "avg_query_seconds": 7.3,
                    "median_query_seconds": 2.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 0.7,
                "top_p": 0.99,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Reflection",
                "harness": null
            }
        },
        "mysterymechanism": {
            "openai/gpt-6-astra": {
                "accuracy": 53.153,
                "latency": 488.597,
                "stderr": 3.357,
                "cost_per_test": 1.559704,
                "token_totals": {
                    "input_tokens": 34970642,
                    "output_tokens": 4706731,
                    "reasoning_tokens": 3583655,
                    "cache_read_tokens": 28320058,
                    "cache_write_tokens": 6436778
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 12.2,
                    "avg_query_seconds": 39.9,
                    "median_query_seconds": 24.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-6.2-astra": {
                "accuracy": 53.153,
                "latency": 634.079,
                "stderr": 3.357,
                "cost_per_test": 1.720342,
                "token_totals": {
                    "input_tokens": 126929498,
                    "output_tokens": 5947890,
                    "reasoning_tokens": 4195258,
                    "cache_read_tokens": 117171758,
                    "cache_write_tokens": 9534889
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 25.8,
                    "avg_query_seconds": 24.5,
                    "median_query_seconds": 15.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-fable-5-2": {
                "accuracy": 51.802,
                "latency": 292.059,
                "stderr": 3.361,
                "cost_per_test": 1.552812,
                "token_totals": {
                    "input_tokens": 67372260,
                    "output_tokens": 6166948,
                    "reasoning_tokens": 4482915,
                    "cache_read_tokens": 58740942,
                    "cache_write_tokens": 8623824
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 15.9,
                    "avg_query_seconds": 18.4,
                    "median_query_seconds": 7.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-opus-5-5": {
                "accuracy": 49.55,
                "latency": 1178.454,
                "stderr": 3.363,
                "cost_per_test": 4.061695,
                "token_totals": {
                    "input_tokens": 658569339,
                    "output_tokens": 29182250,
                    "reasoning_tokens": 23537093,
                    "cache_read_tokens": 619837485,
                    "cache_write_tokens": 38715822
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 35.1,
                    "avg_query_seconds": 33.6,
                    "median_query_seconds": 20.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-sonnet-5-5": {
                "accuracy": 49.099,
                "latency": 1410.236,
                "stderr": 3.363,
                "cost_per_test": 2.984657,
                "token_totals": {
                    "input_tokens": 1091907970,
                    "output_tokens": 42938841,
                    "reasoning_tokens": 35532465,
                    "cache_read_tokens": 1040231244,
                    "cache_write_tokens": 51657746
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 41.7,
                    "avg_query_seconds": 33.8,
                    "median_query_seconds": 21.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6.1-sol": {
                "accuracy": 46.396,
                "latency": 437.025,
                "stderr": 3.355,
                "cost_per_test": 0.284881,
                "token_totals": {
                    "input_tokens": 32268619,
                    "output_tokens": 4481195,
                    "reasoning_tokens": 3364816,
                    "cache_read_tokens": 25888801,
                    "cache_write_tokens": 6166336
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 11.8,
                    "avg_query_seconds": 37.2,
                    "median_query_seconds": 24.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 45.495,
                "latency": 1494.947,
                "stderr": 3.35,
                "cost_per_test": 4.594426,
                "token_totals": {
                    "input_tokens": 901582710,
                    "output_tokens": 21508801,
                    "reasoning_tokens": 10866656,
                    "cache_read_tokens": 793827453,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 52.7,
                    "avg_query_seconds": 28.4,
                    "median_query_seconds": 25.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "google/cobalt-c-high-1p00": {
                "accuracy": 45.045,
                "latency": 1893.321,
                "stderr": 3.347,
                "cost_per_test": 6.630049,
                "token_totals": {
                    "input_tokens": 1648513346,
                    "output_tokens": 57687367,
                    "reasoning_tokens": 30508204,
                    "cache_read_tokens": 1522448968,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 47.2,
                    "avg_query_seconds": 40.1,
                    "median_query_seconds": 37.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 1048576,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-opus-5": {
                "accuracy": 37.387,
                "latency": 1012.543,
                "stderr": 3.255,
                "cost_per_test": 3.280693,
                "token_totals": {
                    "input_tokens": 260014806,
                    "output_tokens": 18050903,
                    "reasoning_tokens": 13538433,
                    "cache_read_tokens": 234441635,
                    "cache_write_tokens": 25563677
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 21.4,
                    "avg_query_seconds": 47.4,
                    "median_query_seconds": 29.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6.2-sol": {
                "accuracy": 36.937,
                "latency": 654.39,
                "stderr": 3.247,
                "cost_per_test": 0.486648,
                "token_totals": {
                    "input_tokens": 120989324,
                    "output_tokens": 6676485,
                    "reasoning_tokens": 4348885,
                    "cache_read_tokens": 108788414,
                    "cache_write_tokens": 11980852
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 21.6,
                    "avg_query_seconds": 30.2,
                    "median_query_seconds": 16.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 36.486,
                "latency": 322.237,
                "stderr": 3.238,
                "cost_per_test": 1.731041,
                "token_totals": {
                    "input_tokens": 827206914,
                    "output_tokens": 17858080,
                    "reasoning_tokens": 9860899,
                    "cache_read_tokens": 733670345,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 70.6,
                    "avg_query_seconds": 4.6,
                    "median_query_seconds": 3.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-haiku-5-5": {
                "accuracy": 35.135,
                "latency": 805.031,
                "stderr": 3.211,
                "cost_per_test": 0.541024,
                "token_totals": {
                    "input_tokens": 910086657,
                    "output_tokens": 41490334,
                    "reasoning_tokens": 33895469,
                    "cache_read_tokens": 861339525,
                    "cache_write_tokens": 48729800
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 38.0,
                    "avg_query_seconds": 21.2,
                    "median_query_seconds": 13.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "meta/muse-wonder-full": {
                "accuracy": 35.135,
                "latency": 2494.329,
                "stderr": 3.211,
                "cost_per_test": 4.163398,
                "token_totals": {
                    "input_tokens": 707253940,
                    "output_tokens": 22089086,
                    "reasoning_tokens": 14663498,
                    "cache_read_tokens": 617506072,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 39.5,
                    "avg_query_seconds": 63.2,
                    "median_query_seconds": 41.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "openai/gpt-5.6-sol": {
                "accuracy": 33.333,
                "latency": 522.744,
                "stderr": 3.171,
                "cost_per_test": 0.96335,
                "token_totals": {
                    "input_tokens": 130077211,
                    "output_tokens": 6106439,
                    "reasoning_tokens": 3933893,
                    "cache_read_tokens": 121397470,
                    "cache_write_tokens": 8456854
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 25.9,
                    "avg_query_seconds": 20.2,
                    "median_query_seconds": 14.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 30.631,
                "latency": 1674.514,
                "stderr": 3.101,
                "cost_per_test": 1.309658,
                "token_totals": {
                    "input_tokens": 161102384,
                    "output_tokens": 23854775,
                    "reasoning_tokens": 17588077,
                    "cache_read_tokens": 116392832,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 26.4,
                    "avg_query_seconds": 63.4,
                    "median_query_seconds": 42.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "openai/gpt-6-sol": {
                "accuracy": 30.18,
                "latency": 373.426,
                "stderr": 3.088,
                "cost_per_test": 0.460931,
                "token_totals": {
                    "input_tokens": 73446321,
                    "output_tokens": 6771421,
                    "reasoning_tokens": 5398618,
                    "cache_read_tokens": 64736689,
                    "cache_write_tokens": 8491566
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 18.6,
                    "avg_query_seconds": 20.0,
                    "median_query_seconds": 11.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "meta/muse-wonder": {
                "accuracy": 27.477,
                "latency": 1096.935,
                "stderr": 3.003,
                "cost_per_test": 3.389451,
                "token_totals": {
                    "input_tokens": 808874983,
                    "output_tokens": 22652618,
                    "reasoning_tokens": 14192096,
                    "cache_read_tokens": 772656344,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 45.4,
                    "avg_query_seconds": 24.2,
                    "median_query_seconds": 18.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "kimi/kimi-k3.1-pro": {
                "accuracy": 26.126,
                "latency": 359.9,
                "stderr": 2.955,
                "cost_per_test": 0.62244,
                "token_totals": {
                    "input_tokens": 154914034,
                    "output_tokens": 8456997,
                    "reasoning_tokens": 6568961,
                    "cache_read_tokens": 142342400,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 21.7,
                    "avg_query_seconds": 16.6,
                    "median_query_seconds": 11.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "thinkingmachines/inkling-2-small": {
                "accuracy": 25.225,
                "latency": 1016.822,
                "stderr": 2.921,
                "cost_per_test": 0.160403,
                "token_totals": {
                    "input_tokens": 364849677,
                    "output_tokens": 18731789,
                    "reasoning_tokens": 0,
                    "cache_read_tokens": 342878464,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 28.5,
                    "avg_query_seconds": 35.7,
                    "median_query_seconds": 23.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "grok/grok-4.7": {
                "accuracy": 25.225,
                "latency": 1052.564,
                "stderr": 2.921,
                "cost_per_test": 1.841835,
                "token_totals": {
                    "input_tokens": 343810087,
                    "output_tokens": 17892803,
                    "reasoning_tokens": 9064037,
                    "cache_read_tokens": 258912640,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 28.5,
                    "avg_query_seconds": 37.0,
                    "median_query_seconds": 32.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "thinkingmachines/inkling-mini-1117": {
                "accuracy": 22.973,
                "latency": 513.33,
                "stderr": 2.83,
                "cost_per_test": 0.143463,
                "token_totals": {
                    "input_tokens": 555577021,
                    "output_tokens": 23488341,
                    "reasoning_tokens": 0,
                    "cache_read_tokens": 526933120,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 37.5,
                    "avg_query_seconds": 13.7,
                    "median_query_seconds": 7.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "fireworks/ember-1": {
                "accuracy": 22.523,
                "latency": 1215.329,
                "stderr": 2.81,
                "cost_per_test": 1.293505,
                "token_totals": {
                    "input_tokens": 139501369,
                    "output_tokens": 13419785,
                    "reasoning_tokens": 10817707,
                    "cache_read_tokens": 123201009,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 18.0,
                    "avg_query_seconds": 67.3,
                    "median_query_seconds": 33.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Fireworks AI",
                "harness": null
            },
            "reflection/Beam-1.0": {
                "accuracy": 22.072,
                "latency": 144.311,
                "stderr": 2.79,
                "cost_per_test": null,
                "token_totals": {
                    "input_tokens": 211587470,
                    "output_tokens": 8130482,
                    "reasoning_tokens": 4665177,
                    "cache_read_tokens": 194116800,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 40.8,
                    "avg_query_seconds": 3.5,
                    "median_query_seconds": 2.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 0.7,
                "top_p": 0.99,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Reflection",
                "harness": null
            },
            "xiaomi/mimo-v2.6-flash": {
                "accuracy": 21.622,
                "latency": 2107.543,
                "stderr": 2.769,
                "cost_per_test": 0.057497,
                "token_totals": {
                    "input_tokens": 354503482,
                    "output_tokens": 25688926,
                    "reasoning_tokens": 20934111,
                    "cache_read_tokens": 321129792,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 25.2,
                    "avg_query_seconds": 83.7,
                    "median_query_seconds": 38.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "deepseek/deepseek-v4.1-flash": {
                "accuracy": 21.171,
                "latency": 484.327,
                "stderr": 2.748,
                "cost_per_test": 0.164089,
                "turn_metrics": {
                    "tasks_measured": 221,
                    "avg_turns": 36.9,
                    "avg_query_seconds": 13.1,
                    "median_query_seconds": 9.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 384000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "google/gemini-3.1-pro-preview": {
                "accuracy": 20.721,
                "latency": 360.456,
                "stderr": 2.726,
                "cost_per_test": 1.065147,
                "token_totals": {
                    "input_tokens": 262123547,
                    "output_tokens": 8178370,
                    "reasoning_tokens": 3546375,
                    "cache_read_tokens": 214402702,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 46.0,
                    "avg_query_seconds": 7.8,
                    "median_query_seconds": 6.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-6-luna": {
                "accuracy": 19.369,
                "latency": 361.208,
                "stderr": 2.658,
                "cost_per_test": 0.025854,
                "token_totals": {
                    "input_tokens": 66323704,
                    "output_tokens": 8047144,
                    "reasoning_tokens": 7044372,
                    "cache_read_tokens": 57123096,
                    "cache_write_tokens": 8985227
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 14.6,
                    "avg_query_seconds": 24.7,
                    "median_query_seconds": 16.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "stepfun/step-5-preview": {
                "accuracy": 18.468,
                "latency": 1760.418,
                "stderr": 2.61,
                "cost_per_test": 0.597179,
                "token_totals": {
                    "input_tokens": 455746580,
                    "output_tokens": 27980610,
                    "reasoning_tokens": 0,
                    "cache_read_tokens": 419705856,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 27.6,
                    "avg_query_seconds": 63.7,
                    "median_query_seconds": 37.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Stepfun",
                "harness": null
            },
            "openai/gpt-5.6-luna": {
                "accuracy": 14.414,
                "latency": 493.052,
                "stderr": 2.363,
                "cost_per_test": 0.109246,
                "token_totals": {
                    "input_tokens": 447050993,
                    "output_tokens": 9484085,
                    "reasoning_tokens": 5857002,
                    "cache_read_tokens": 431065411,
                    "cache_write_tokens": 15750179
                },
                "token_tasks_measured": 222,
                "turn_metrics": {
                    "tasks_measured": 222,
                    "avg_turns": 44.7,
                    "avg_query_seconds": 11.0,
                    "median_query_seconds": 8.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            }
        },
        "terminal_bench_science": {
            "openai/gpt-6.2-astra": {
                "accuracy": 71.429,
                "latency": 2860.911,
                "stderr": 5.438,
                "cost_per_test": 11.082343,
                "token_totals": {
                    "input_tokens": 641551935,
                    "output_tokens": 6889255,
                    "reasoning_tokens": 4258891,
                    "cache_read_tokens": 628760483,
                    "cache_write_tokens": 12775671
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 65.7,
                    "avg_query_seconds": 39.8,
                    "median_query_seconds": 28.6,
                    "avg_ttft_seconds": 19.8,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/cobalt-c-high-1p00": {
                "accuracy": 62.857,
                "latency": 4359.939,
                "stderr": 5.817,
                "cost_per_test": 13.741423,
                "token_totals": {
                    "input_tokens": 2423786107,
                    "output_tokens": 20377725,
                    "reasoning_tokens": 8820563,
                    "cache_read_tokens": 2321096338,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 109.0,
                    "avg_query_seconds": 23.0,
                    "median_query_seconds": 13.0,
                    "avg_ttft_seconds": 5.9,
                    "output_time_split": {
                        "reasoning_pct": 56.3,
                        "content_pct": 39.8,
                        "tool_call_pct": 3.9
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 1048576,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-6-astra": {
                "accuracy": 62.857,
                "latency": 6534.868,
                "stderr": 5.817,
                "cost_per_test": 20.802802,
                "token_totals": {
                    "input_tokens": 405026379,
                    "output_tokens": 11095298,
                    "reasoning_tokens": 8544230,
                    "cache_read_tokens": 386366671,
                    "cache_write_tokens": 18635815
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 47.1,
                    "avg_query_seconds": 120.1,
                    "median_query_seconds": 65.1,
                    "avg_ttft_seconds": 98.6,
                    "output_time_split": {
                        "reasoning_pct": 2.9,
                        "content_pct": 18.6,
                        "tool_call_pct": 78.5
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-6.2-sol": {
                "accuracy": 58.571,
                "latency": 2343.448,
                "stderr": 5.93,
                "cost_per_test": 2.53223,
                "token_totals": {
                    "input_tokens": 595407273,
                    "output_tokens": 6562776,
                    "reasoning_tokens": 3638428,
                    "cache_read_tokens": 581407439,
                    "cache_write_tokens": 13984386
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 64.1,
                    "avg_query_seconds": 32.2,
                    "median_query_seconds": 20.4,
                    "avg_ttft_seconds": 13.2,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-fable-5-2": {
                "accuracy": 55.714,
                "latency": 5702.569,
                "stderr": 5.98,
                "cost_per_test": 23.13708,
                "token_totals": {
                    "input_tokens": 3112332023,
                    "output_tokens": 16268273,
                    "reasoning_tokens": 10245070,
                    "cache_read_tokens": 3077489383,
                    "cache_write_tokens": 34820080
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 160.7,
                    "avg_query_seconds": 16.9,
                    "median_query_seconds": 8.3,
                    "avg_ttft_seconds": 16.7,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6.1-sol": {
                "accuracy": 52.857,
                "latency": 5003.987,
                "stderr": 6.009,
                "cost_per_test": 3.436948,
                "token_totals": {
                    "input_tokens": 337815176,
                    "output_tokens": 11533409,
                    "reasoning_tokens": 9185469,
                    "cache_read_tokens": 317832491,
                    "cache_write_tokens": 19972804
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 42.1,
                    "avg_query_seconds": 130.2,
                    "median_query_seconds": 61.4,
                    "avg_ttft_seconds": 106.4,
                    "output_time_split": {
                        "reasoning_pct": 4.0,
                        "content_pct": 11.1,
                        "tool_call_pct": 84.9
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-opus-5-5": {
                "accuracy": 47.143,
                "latency": 8856.116,
                "stderr": 6.009,
                "cost_per_test": 19.121966,
                "token_totals": {
                    "input_tokens": 1829311051,
                    "output_tokens": 17393519,
                    "reasoning_tokens": 11843007,
                    "cache_read_tokens": 1700954981,
                    "cache_write_tokens": 128339444
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 117.3,
                    "avg_query_seconds": 22.9,
                    "median_query_seconds": 14.2,
                    "avg_ttft_seconds": 21.8,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-sonnet-5-5": {
                "accuracy": 47.143,
                "latency": 9980.148,
                "stderr": 6.009,
                "cost_per_test": 19.568918,
                "token_totals": {
                    "input_tokens": 9111131259,
                    "output_tokens": 34342805,
                    "reasoning_tokens": 24263221,
                    "cache_read_tokens": 9063088192,
                    "cache_write_tokens": 48002507
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 285.1,
                    "avg_query_seconds": 16.6,
                    "median_query_seconds": 10.6,
                    "avg_ttft_seconds": 16.4,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 44.286,
                "latency": 6369.095,
                "stderr": 5.98,
                "cost_per_test": 44.904641,
                "token_totals": {
                    "input_tokens": 2230737485,
                    "output_tokens": 21755185,
                    "reasoning_tokens": 10397231,
                    "cache_read_tokens": 1635454933,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 113.1,
                    "avg_query_seconds": 47.3,
                    "median_query_seconds": 35.2,
                    "avg_ttft_seconds": 14.4,
                    "output_time_split": {
                        "reasoning_pct": 52.8,
                        "content_pct": 46.9,
                        "tool_call_pct": 0.3
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "meta/muse-wonder-full": {
                "accuracy": 37.143,
                "latency": 12919.209,
                "stderr": 5.817,
                "cost_per_test": 22.814086,
                "token_totals": {
                    "input_tokens": 3713083934,
                    "output_tokens": 17062678,
                    "reasoning_tokens": 11556867,
                    "cache_read_tokens": 3578053496,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 191.6,
                    "avg_query_seconds": 54.4,
                    "median_query_seconds": 20.5,
                    "avg_ttft_seconds": 52.5,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "kimi/kimi-k3.1-pro": {
                "accuracy": 31.429,
                "latency": 3474.879,
                "stderr": 5.589,
                "cost_per_test": 8.577155,
                "token_totals": {
                    "input_tokens": 2310199935,
                    "output_tokens": 10019167,
                    "reasoning_tokens": 5787120,
                    "cache_read_tokens": 2288994816,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 163.5,
                    "avg_query_seconds": 10.3,
                    "median_query_seconds": 5.8,
                    "avg_ttft_seconds": 2.6,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "openai/gpt-6-sol": {
                "accuracy": 30.0,
                "latency": 4930.428,
                "stderr": 5.517,
                "cost_per_test": 5.821477,
                "token_totals": {
                    "input_tokens": 1106517592,
                    "output_tokens": 9142533,
                    "reasoning_tokens": 5482665,
                    "cache_read_tokens": 1090637986,
                    "cache_write_tokens": 15843344
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 105.6,
                    "avg_query_seconds": 26.1,
                    "median_query_seconds": 15.6,
                    "avg_ttft_seconds": 13.7,
                    "output_time_split": {
                        "reasoning_pct": 12.8,
                        "content_pct": 5.8,
                        "tool_call_pct": 81.3
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-opus-5": {
                "accuracy": 27.143,
                "latency": 7908.453,
                "stderr": 5.354,
                "cost_per_test": 32.543386,
                "token_totals": {
                    "input_tokens": 3251326101,
                    "output_tokens": 18648623,
                    "reasoning_tokens": 11809096,
                    "cache_read_tokens": 3218944868,
                    "cache_write_tokens": 32354267
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 192.6,
                    "avg_query_seconds": 20.2,
                    "median_query_seconds": 9.2,
                    "avg_ttft_seconds": 19.0,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "meta/muse-wonder": {
                "accuracy": 22.857,
                "latency": 5584.198,
                "stderr": 5.055,
                "cost_per_test": 19.609644,
                "token_totals": {
                    "input_tokens": 4434041175,
                    "output_tokens": 17847376,
                    "reasoning_tokens": 12044271,
                    "cache_read_tokens": 4400115036,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 238.7,
                    "avg_query_seconds": 13.5,
                    "median_query_seconds": 7.1,
                    "avg_ttft_seconds": 12.3,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "openai/gpt-5.6-sol": {
                "accuracy": 20.0,
                "latency": 6216.027,
                "stderr": 4.815,
                "cost_per_test": 7.322071,
                "token_totals": {
                    "input_tokens": 662422169,
                    "output_tokens": 6024299,
                    "reasoning_tokens": 2789570,
                    "cache_read_tokens": 648354535,
                    "cache_write_tokens": 14038242
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 72.8,
                    "avg_query_seconds": 19.7,
                    "median_query_seconds": 13.1,
                    "avg_ttft_seconds": 10.0,
                    "output_time_split": {
                        "reasoning_pct": 14.9,
                        "content_pct": 26.8,
                        "tool_call_pct": 58.3
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 11.429,
                "latency": 4696.051,
                "stderr": 3.83,
                "cost_per_test": 4.815933,
                "token_totals": {
                    "input_tokens": 519819247,
                    "output_tokens": 7158015,
                    "reasoning_tokens": 3485074,
                    "cache_read_tokens": 496980864,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 65.2,
                    "avg_query_seconds": 24.2,
                    "median_query_seconds": 13.9,
                    "avg_ttft_seconds": 4.8,
                    "output_time_split": {
                        "reasoning_pct": 79.3,
                        "content_pct": 20.7,
                        "tool_call_pct": 0.0
                    }
                },
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "grok/grok-4.7": {
                "accuracy": 10.0,
                "latency": 5553.191,
                "stderr": 3.612,
                "cost_per_test": 17.040638,
                "token_totals": {
                    "input_tokens": 1158107075,
                    "output_tokens": 11957556,
                    "reasoning_tokens": 6886200,
                    "cache_read_tokens": 1084056704,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 93.7,
                    "avg_query_seconds": 26.6,
                    "median_query_seconds": 18.4,
                    "avg_ttft_seconds": 3.7,
                    "output_time_split": {
                        "reasoning_pct": 94.4,
                        "content_pct": 5.6,
                        "tool_call_pct": 0.0
                    }
                },
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "anthropic/claude-haiku-5-5": {
                "accuracy": 10.0,
                "latency": 5637.472,
                "stderr": 3.612,
                "cost_per_test": 4.523698,
                "token_totals": {
                    "input_tokens": 4964740201,
                    "output_tokens": 26152873,
                    "reasoning_tokens": 19018280,
                    "cache_read_tokens": 4931170604,
                    "cache_write_tokens": 33540337
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 208.4,
                    "avg_query_seconds": 9.0,
                    "median_query_seconds": 4.1,
                    "avg_ttft_seconds": 7.6,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 8.571,
                "latency": 3311.435,
                "stderr": 3.37,
                "cost_per_test": 5.643008,
                "token_totals": {
                    "input_tokens": 1352310020,
                    "output_tokens": 11150231,
                    "reasoning_tokens": 5862209,
                    "cache_read_tokens": 1271911976,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 69,
                    "avg_turns": 146.3,
                    "avg_query_seconds": 5.8,
                    "median_query_seconds": 4.2,
                    "avg_ttft_seconds": 3.3,
                    "output_time_split": {
                        "reasoning_pct": 97.9,
                        "content_pct": 0.9,
                        "tool_call_pct": 1.3
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "fireworks/ember-1": {
                "accuracy": 5.714,
                "latency": 12196.691,
                "stderr": 2.794,
                "cost_per_test": 30.760832,
                "token_totals": {
                    "input_tokens": 5574642539,
                    "output_tokens": 19999519,
                    "reasoning_tokens": 13579586,
                    "cache_read_tokens": 5507652660,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 276.7,
                    "avg_query_seconds": 23.4,
                    "median_query_seconds": 13.2,
                    "avg_ttft_seconds": 6.4,
                    "output_time_split": {
                        "reasoning_pct": 71.1,
                        "content_pct": 1.7,
                        "tool_call_pct": 27.2
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Fireworks AI",
                "harness": null
            },
            "openai/gpt-6-luna": {
                "accuracy": 4.286,
                "latency": 4559.47,
                "stderr": 2.438,
                "cost_per_test": 0.240966,
                "token_totals": {
                    "input_tokens": 771110577,
                    "output_tokens": 9956631,
                    "reasoning_tokens": 7497116,
                    "cache_read_tokens": 755113085,
                    "cache_write_tokens": 15962745
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 76.0,
                    "avg_query_seconds": 20.7,
                    "median_query_seconds": 10.9,
                    "avg_ttft_seconds": 7.2,
                    "output_time_split": {
                        "reasoning_pct": 47.6,
                        "content_pct": 5.1,
                        "tool_call_pct": 47.3
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "xiaomi/mimo-v2.6-flash": {
                "accuracy": 4.286,
                "latency": 14326.966,
                "stderr": 2.438,
                "cost_per_test": 0.244346,
                "token_totals": {
                    "input_tokens": 2120700869,
                    "output_tokens": 19847075,
                    "reasoning_tokens": 14752104,
                    "cache_read_tokens": 2079818560,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 125.8,
                    "avg_query_seconds": 52.4,
                    "median_query_seconds": 29.4,
                    "avg_ttft_seconds": 13.4,
                    "output_time_split": {
                        "reasoning_pct": 85.9,
                        "content_pct": 0.8,
                        "tool_call_pct": 13.3
                    }
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "thinkingmachines/inkling-2-small": {
                "accuracy": 2.857,
                "latency": 8862.875,
                "stderr": 2.006,
                "cost_per_test": 5.546533,
                "token_totals": {
                    "input_tokens": 14316616653,
                    "output_tokens": 32762714,
                    "reasoning_tokens": null,
                    "cache_read_tokens": 14269235968,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 394.4,
                    "avg_query_seconds": 10.4,
                    "median_query_seconds": 5.3,
                    "avg_ttft_seconds": 4.2,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "stepfun/step-5-preview": {
                "accuracy": 2.857,
                "latency": 8997.76,
                "stderr": 2.006,
                "cost_per_test": 4.250933,
                "token_totals": {
                    "input_tokens": 4101639238,
                    "output_tokens": 21535694,
                    "reasoning_tokens": 0,
                    "cache_read_tokens": 4065495040,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 217.8,
                    "avg_query_seconds": 23.0,
                    "median_query_seconds": 11.4,
                    "avg_ttft_seconds": 5.2,
                    "output_time_split": {
                        "reasoning_pct": 80.3,
                        "content_pct": 0.4,
                        "tool_call_pct": 19.3
                    }
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Stepfun",
                "harness": null
            },
            "google/gemini-3.1-pro-preview": {
                "accuracy": 1.429,
                "latency": 2638.065,
                "stderr": 1.429,
                "cost_per_test": 5.907095,
                "token_totals": {
                    "input_tokens": 858729252,
                    "output_tokens": 3958785,
                    "reasoning_tokens": 2054169,
                    "cache_read_tokens": 796983036,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 122.2,
                    "avg_query_seconds": 12.0,
                    "median_query_seconds": 7.1,
                    "avg_ttft_seconds": 9.7,
                    "output_time_split": {
                        "reasoning_pct": 84.7,
                        "content_pct": 14.3,
                        "tool_call_pct": 1.0
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "reflection/Beam-1.0": {
                "accuracy": 1.429,
                "latency": 4658.837,
                "stderr": 1.429,
                "cost_per_test": null,
                "token_totals": {
                    "input_tokens": 1817962438,
                    "output_tokens": 4532859,
                    "reasoning_tokens": 2389890,
                    "cache_read_tokens": 1800613696,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 260.8,
                    "avg_query_seconds": 3.5,
                    "median_query_seconds": 1.3,
                    "avg_ttft_seconds": 2.9,
                    "output_time_split": null
                },
                "temperature": 0.7,
                "top_p": 0.99,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Reflection",
                "harness": null
            },
            "thinkingmachines/inkling-mini-1117": {
                "accuracy": 0.0,
                "latency": 7122.697,
                "stderr": 0.0,
                "cost_per_test": 4.572654,
                "token_totals": {
                    "input_tokens": 10127998739,
                    "output_tokens": 24205159,
                    "reasoning_tokens": null,
                    "cache_read_tokens": 10093471872,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 371.7,
                    "avg_query_seconds": 5.9,
                    "median_query_seconds": 2.8,
                    "avg_ttft_seconds": 2.9,
                    "output_time_split": {
                        "reasoning_pct": 86.4,
                        "content_pct": 13.3,
                        "tool_call_pct": 0.4
                    }
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 1024000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "openai/gpt-5.6-luna": {
                "accuracy": 0.0,
                "latency": 7396.452,
                "stderr": 0.0,
                "cost_per_test": 0.581456,
                "token_totals": {
                    "input_tokens": 1129716462,
                    "output_tokens": 7266560,
                    "reasoning_tokens": 3921207,
                    "cache_read_tokens": 1113668160,
                    "cache_write_tokens": 16009244
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 120.9,
                    "avg_query_seconds": 10.7,
                    "median_query_seconds": 6.9,
                    "avg_ttft_seconds": 5.7,
                    "output_time_split": {
                        "reasoning_pct": 22.5,
                        "content_pct": 1.4,
                        "tool_call_pct": 76.1
                    }
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "deepseek/deepseek-v4.1-flash": {
                "accuracy": 0.0,
                "latency": 10621.573,
                "stderr": 0.0,
                "cost_per_test": 0.473154,
                "token_totals": {
                    "input_tokens": 1765162289,
                    "output_tokens": 17312542,
                    "reasoning_tokens": 12785452,
                    "cache_read_tokens": 1759193848,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 70,
                "turn_metrics": {
                    "tasks_measured": 70,
                    "avg_turns": 136.1,
                    "avg_query_seconds": 10.0,
                    "median_query_seconds": 7.1,
                    "avg_ttft_seconds": 2.3,
                    "output_time_split": {
                        "reasoning_pct": 82.0,
                        "content_pct": 0.4,
                        "tool_call_pct": 17.6
                    }
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 384000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            }
        }
    }
}