{
    "metadata": {
        "benchmark": "MedScribe",
        "slug": "medscribe",
        "description": "Can models support doctors with their administrative work?",
        "benchmark_id": "medscribe",
        "family": "medscribe",
        "version": "1",
        "updated": "2026-10-29",
        "dataset_type": "private",
        "industry": "healthcare",
        "tasks": {
            "overall": "Overall"
        },
        "models": [
            "alibaba/qwen3-max-2026-01-23",
            "alibaba/qwen3-vl-plus-2025-09-23",
            "alibaba/qwen3.5-flash",
            "alibaba/qwen3.6-plus",
            "alibaba/qwen3.7-max",
            "alibaba/qwen3.8-27b",
            "alibaba/qwen3.8-max",
            "ant/ling-3.0-flash-2607",
            "ant/ling-3.0-flash-af-rc3",
            "anthropic/claude-fable-5",
            "anthropic/claude-fable-5-1",
            "anthropic/claude-haiku-4-5-20251001-thinking",
            "anthropic/claude-haiku-5-5",
            "anthropic/claude-opus-4-1-20250805",
            "anthropic/claude-opus-4-1-20250805-thinking",
            "anthropic/claude-opus-4-5-20251101",
            "anthropic/claude-opus-4-5-20251101-thinking",
            "anthropic/claude-opus-4-6",
            "anthropic/claude-opus-4-6-thinking",
            "anthropic/claude-opus-4-7",
            "anthropic/claude-opus-4-8",
            "anthropic/claude-opus-5",
            "anthropic/claude-opus-5-5",
            "anthropic/claude-sonnet-4-20250514",
            "anthropic/claude-sonnet-4-20250514-thinking",
            "anthropic/claude-sonnet-4-5-20250929",
            "anthropic/claude-sonnet-4-5-20250929-thinking",
            "anthropic/claude-sonnet-5",
            "anthropic/claude-sonnet-5-5",
            "cohere/command-a-plus-05-2026",
            "deepseek/deepseek-v4-flash-0731",
            "deepseek/deepseek-v4-pro",
            "deepseek/deepseek-v4-pro-0813",
            "deepseek/deepseek-v4.1-flash",
            "fireworks/llama4-maverick-instruct-basic",
            "fireworks/nemotron-lightning-3p5-30b-a3b",
            "google/gemini-2.5-flash",
            "google/gemini-2.5-flash-lite",
            "google/gemini-2.5-flash-lite-preview-09-2025",
            "google/gemini-2.5-flash-lite-preview-09-2025-thinking",
            "google/gemini-2.5-flash-preview-09-2025",
            "google/gemini-2.5-flash-preview-09-2025-thinking",
            "google/gemini-2.5-flash-thinking",
            "google/gemini-2.5-pro",
            "google/gemini-3-flash-preview",
            "google/gemini-3-pro-preview",
            "google/gemini-3.1-flash-lite-preview",
            "google/gemini-3.1-pro-preview",
            "google/gemini-3.5-flash",
            "google/gemini-3.5-flash-lite",
            "google/gemini-3.6-flash",
            "google/gemini-3.7-flash",
            "google/gemini-3.8-flash",
            "google/gemini-4-argon",
            "grok/grok-4-0709",
            "grok/grok-4-1-fast-non-reasoning",
            "grok/grok-4-1-fast-reasoning",
            "grok/grok-4-fast-non-reasoning",
            "grok/grok-4-fast-reasoning",
            "grok/grok-4.20-0309-reasoning",
            "grok/grok-4.3",
            "grok/grok-4.5",
            "grok/grok-4.6",
            "grok/grok-4.7",
            "inception/mercury-2.5",
            "kimi/kimi-k2.5-thinking",
            "kimi/kimi-k2.6",
            "kimi/kimi-k3",
            "meta/muse-wonder-full",
            "meta/muse_spark",
            "meta/muse_spark_1_1",
            "meta/muse_spark_1_2",
            "minimax/MiniMax-M2.1",
            "minimax/MiniMax-M2.7",
            "minimax/MiniMax-M3",
            "mistralai/mistral-large-4",
            "mistralai/mistral-medium-3.5",
            "openai/gpt-5-2025-08-07",
            "openai/gpt-5-mini-2025-08-07",
            "openai/gpt-5-nano-2025-08-07",
            "openai/gpt-5.1-2025-11-13",
            "openai/gpt-5.2-2025-12-11",
            "openai/gpt-5.4-2026-03-05",
            "openai/gpt-5.4-nano-2026-03-17",
            "openai/gpt-5.5",
            "openai/gpt-5.6-luna",
            "openai/gpt-5.6-sol",
            "openai/gpt-5.6-terra",
            "openai/gpt-6-astra",
            "openai/gpt-6-luna",
            "openai/gpt-6-sol",
            "openai/gpt-6.1-sol",
            "openai/o3-2025-04-16",
            "openai/o4-mini-2025-04-16",
            "poolside/laguna-m.1",
            "poolside/laguna-xs.2",
            "tencent/hy4-preview",
            "thinkingmachines/inkling",
            "thinkingmachines/inkling-small",
            "together/meta-llama/Llama-4-Scout-17B-16E-Instruct",
            "xiaomi/mimo-v2.5",
            "xiaomi/mimo-v2.5-pro",
            "xiaomi/mimo-v2.6-flash",
            "xiaomi/mimo-v2.6-pro",
            "zai/glm-4.7",
            "zai/glm-5.1",
            "zai/glm-5.2",
            "zai/glm-5.3",
            "zai/glm-5.3-flash"
        ],
        "partners": [],
        "showBadge": false,
        "visible": true,
        "use_cost_per_test": false,
        "runner": "platform",
        "mode": "one-shot",
        "archived": false,
        "partner": false,
        "total_models": 109
    },
    "tasks": {
        "overall": {
            "anthropic/claude-opus-5-5": {
                "accuracy": 91.43,
                "latency": 441.582,
                "stderr": 1.932,
                "cost_per_test": 1.154156,
                "avg_input_tokens": 9688.5,
                "avg_output_tokens": 55678.4,
                "avg_reasoning_tokens": 50463.3,
                "avg_cache_read_tokens": 1635.3,
                "avg_cache_write_tokens": 8049.1,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 441.6,
                    "median_query_seconds": 424.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-fable-5-1": {
                "accuracy": 91.294,
                "latency": 188.363,
                "stderr": 1.953,
                "cost_per_test": 0.9635,
                "avg_input_tokens": 9688.5,
                "avg_output_tokens": 17526.1,
                "avg_reasoning_tokens": 12091.0,
                "avg_cache_read_tokens": 2767.5,
                "avg_cache_write_tokens": 6917.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 188.4,
                    "median_query_seconds": 191.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-sonnet-5-5": {
                "accuracy": 91.101,
                "latency": 306.811,
                "stderr": 1.96,
                "cost_per_test": 0.508604,
                "avg_input_tokens": 9688.5,
                "avg_output_tokens": 48901.4,
                "avg_reasoning_tokens": 44314.0,
                "avg_cache_read_tokens": 2012.7,
                "avg_cache_write_tokens": 7671.8,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 306.8,
                    "median_query_seconds": 315.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-opus-5": {
                "accuracy": 90.985,
                "latency": 76.563,
                "stderr": 1.916,
                "cost_per_test": 0.236275,
                "avg_input_tokens": 9686.5,
                "avg_output_tokens": 7058.4,
                "avg_cache_read_tokens": 125.8,
                "avg_cache_write_tokens": 9558.7,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 76.6,
                    "median_query_seconds": 73.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "meta/muse_spark_1_2": {
                "accuracy": 90.062,
                "latency": 61.458,
                "stderr": 1.958,
                "cost_per_test": 0.037779,
                "avg_input_tokens": 6122.7,
                "avg_output_tokens": 7428.7,
                "avg_reasoning_tokens": 4770.3,
                "avg_cache_read_tokens": 1315.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 61.5,
                    "median_query_seconds": 62.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "anthropic/claude-haiku-5-5": {
                "accuracy": 89.542,
                "latency": 323.396,
                "stderr": 1.917,
                "cost_per_test": 0.043777,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "grok/grok-4.7": {
                "accuracy": 89.377,
                "latency": 153.053,
                "stderr": 1.886,
                "cost_per_test": 0.074505,
                "avg_input_tokens": 7306.1,
                "avg_output_tokens": 10289.7,
                "avg_reasoning_tokens": 8189.8,
                "avg_cache_read_tokens": 1230.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 153.1,
                    "median_query_seconds": 149.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "meta/muse-wonder-full": {
                "accuracy": 89.335,
                "latency": 115.718,
                "stderr": 2.004,
                "cost_per_test": 0.19181,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "zai/glm-5.3-flash": {
                "accuracy": 88.936,
                "latency": 86.98,
                "stderr": 1.907,
                "cost_per_test": 0.002235,
                "avg_input_tokens": 6308.8,
                "avg_output_tokens": 7224.3,
                "avg_cache_read_tokens": 731.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 87.0,
                    "median_query_seconds": 84.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "meta/muse_spark_1_1": {
                "accuracy": 88.888,
                "latency": 63.335,
                "stderr": 1.95,
                "cost_per_test": 0.034628,
                "avg_input_tokens": 6265.7,
                "avg_output_tokens": 7611.6,
                "avg_reasoning_tokens": 5227.3,
                "avg_cache_read_tokens": 5048.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 63.3,
                    "median_query_seconds": 58.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "zai/glm-5.3": {
                "accuracy": 88.81,
                "latency": 121.931,
                "stderr": 1.999,
                "cost_per_test": 0.057236,
                "avg_input_tokens": 6308.8,
                "avg_output_tokens": 11083.7,
                "avg_cache_read_tokens": 320.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 121.9,
                    "median_query_seconds": 108.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "anthropic/claude-fable-5": {
                "accuracy": 88.522,
                "latency": 119.475,
                "stderr": 1.945,
                "cost_per_test": 0.583239,
                "avg_input_tokens": 9686.5,
                "avg_output_tokens": 9339.5,
                "avg_cache_read_tokens": 418.3,
                "avg_cache_write_tokens": 9266.1,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 119.5,
                    "median_query_seconds": 119.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "xiaomi/mimo-v2.6-pro": {
                "accuracy": 88.307,
                "latency": 173.345,
                "stderr": 1.938,
                "cost_per_test": 0.00988,
                "avg_input_tokens": 6338.2,
                "avg_output_tokens": 8187.4,
                "avg_reasoning_tokens": 5585.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 173.3,
                    "median_query_seconds": 149.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "openai/gpt-5.1-2025-11-13": {
                "accuracy": 88.09,
                "latency": 77.983,
                "stderr": 1.942,
                "cost_per_test": 0.096508,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 9076.4,
                "avg_reasoning_tokens": 6431.5,
                "avg_cache_read_tokens": 1682.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 78.0,
                    "median_query_seconds": 73.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "kimi/kimi-k3": {
                "accuracy": 88.048,
                "latency": 38.241,
                "stderr": 1.976,
                "cost_per_test": 0.110262,
                "avg_input_tokens": 6237.6,
                "avg_output_tokens": 6837.8,
                "avg_reasoning_tokens": 4818.8,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 38.2,
                    "median_query_seconds": 36.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "openai/gpt-6-astra": {
                "accuracy": 87.908,
                "latency": 153.728,
                "stderr": 1.938,
                "cost_per_test": 0.581991,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 10712.7,
                "avg_reasoning_tokens": 9038.9,
                "avg_cache_read_tokens": 1636.6,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 153.7,
                    "median_query_seconds": 147.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 87.427,
                "latency": 87.439,
                "stderr": 1.964,
                "cost_per_test": 0.147175,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 6039.8,
                "avg_reasoning_tokens": 4277.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 87.4,
                    "median_query_seconds": 92.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "minimax/MiniMax-M3": {
                "accuracy": 87.252,
                "latency": 123.967,
                "stderr": 1.957,
                "cost_per_test": 0.013748,
                "avg_input_tokens": 6184.8,
                "avg_output_tokens": 4443.1,
                "avg_cache_read_tokens": 1304.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 124.0,
                    "median_query_seconds": 113.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "MiniMax",
                "harness": null
            },
            "grok/grok-4.5": {
                "accuracy": 86.884,
                "latency": 704.513,
                "stderr": 1.944,
                "cost_per_test": 0.033208,
                "avg_input_tokens": 6094.1,
                "avg_output_tokens": 3869.1,
                "avg_reasoning_tokens": 2180.1,
                "avg_cache_read_tokens": 1462.9,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 704.5,
                    "median_query_seconds": 698.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "openai/gpt-5.5": {
                "accuracy": 86.868,
                "latency": 132.761,
                "stderr": 1.932,
                "cost_per_test": 0.142988,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 7093.3,
                "avg_reasoning_tokens": 5029.5,
                "avg_cache_read_tokens": 1118.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 132.8,
                    "median_query_seconds": 123.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-opus-4-6": {
                "accuracy": 86.738,
                "latency": 54.318,
                "stderr": 1.942,
                "cost_per_test": 0.115121,
                "avg_input_tokens": 6913.7,
                "avg_output_tokens": 2876.6,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_write_tokens": 6910.7,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 54.3,
                    "median_query_seconds": 50.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 86.534,
                "latency": 70.944,
                "stderr": 1.956,
                "cost_per_test": 0.03719,
                "avg_input_tokens": 6094.1,
                "avg_output_tokens": 4176.1,
                "avg_reasoning_tokens": 2657.9,
                "avg_cache_read_tokens": 36.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 70.9,
                    "median_query_seconds": 70.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "openai/gpt-6.1-sol": {
                "accuracy": 86.453,
                "latency": 169.619,
                "stderr": 1.898,
                "cost_per_test": 0.100902,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 8992.5,
                "avg_reasoning_tokens": 7392.9,
                "avg_cache_read_tokens": 1788.9,
                "avg_cache_write_tokens": 4316.9,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 169.6,
                    "median_query_seconds": 158.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-opus-4-6-thinking": {
                "accuracy": 86.13,
                "latency": 127.564,
                "stderr": 1.944,
                "cost_per_test": 0.224735,
                "avg_input_tokens": 6931.7,
                "avg_output_tokens": 7347.2,
                "avg_cache_read_tokens": 393.9,
                "avg_cache_write_tokens": 6534.8,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 127.6,
                    "median_query_seconds": 121.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "meta/muse_spark": {
                "accuracy": 85.902,
                "latency": 191.473,
                "stderr": 1.847,
                "cost_per_test": 0.007681,
                "avg_input_tokens": 6159.7,
                "avg_output_tokens": 11261.6,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 191.5,
                    "median_query_seconds": 189.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "anthropic/claude-opus-4-8": {
                "accuracy": 85.755,
                "latency": 82.364,
                "stderr": 1.928,
                "cost_per_test": 0.259121,
                "avg_input_tokens": 9686.5,
                "avg_output_tokens": 7943.3,
                "avg_cache_write_tokens": 9684.5,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 82.4,
                    "median_query_seconds": 83.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "deepseek/deepseek-v4.1-flash": {
                "accuracy": 85.5,
                "latency": 53.826,
                "stderr": 1.918,
                "cost_per_test": 0.015397,
                "avg_input_tokens": 6263.4,
                "avg_output_tokens": 11656.2,
                "avg_reasoning_tokens": 9770.1,
                "avg_cache_read_tokens": 1596.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 53.8,
                    "median_query_seconds": 50.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 384000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "thinkingmachines/inkling": {
                "accuracy": 85.405,
                "latency": 244.954,
                "stderr": 1.844,
                "cost_per_test": 0.165561,
                "avg_input_tokens": 6115.7,
                "avg_output_tokens": 15762.2,
                "avg_cache_read_tokens": 1726.7,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 245.0,
                    "median_query_seconds": 251.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "anthropic/claude-opus-4-5-20251101-thinking": {
                "accuracy": 85.321,
                "latency": 72.112,
                "stderr": 1.896,
                "cost_per_test": 0.410224,
                "avg_input_tokens": 6943.7,
                "avg_output_tokens": 3768.7,
                "avg_cache_read_tokens": 149.8,
                "avg_cache_write_tokens": 6783.9,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 72.1,
                    "median_query_seconds": 69.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "xiaomi/mimo-v2.6-flash": {
                "accuracy": 85.275,
                "latency": 74.481,
                "stderr": 1.986,
                "cost_per_test": 0.002184,
                "avg_input_tokens": 6338.2,
                "avg_output_tokens": 4629.3,
                "avg_reasoning_tokens": 2556.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 74.5,
                    "median_query_seconds": 51.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "openai/gpt-5.6-sol": {
                "accuracy": 85.233,
                "latency": 94.404,
                "stderr": 1.973,
                "cost_per_test": 0.276691,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 8204.9,
                "avg_reasoning_tokens": 6666.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 94.4,
                    "median_query_seconds": 84.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-haiku-4-5-20251001-thinking": {
                "accuracy": 85.23,
                "latency": 66.197,
                "stderr": 1.899,
                "cost_per_test": 0.042375,
                "avg_input_tokens": 6943.7,
                "avg_output_tokens": 6739.7,
                "avg_cache_write_tokens": 6933.7,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 66.2,
                    "median_query_seconds": 64.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "alibaba/qwen3.8-max": {
                "accuracy": 84.947,
                "latency": 251.26,
                "stderr": 1.999,
                "cost_per_test": 0.089616,
                "avg_input_tokens": 6592.2,
                "avg_output_tokens": 12738.6,
                "avg_reasoning_tokens": 10678.8,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 251.3,
                    "median_query_seconds": 252.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "anthropic/claude-sonnet-4-5-20250929": {
                "accuracy": 84.515,
                "latency": 44.419,
                "stderr": 1.929,
                "cost_per_test": 0.054649,
                "avg_input_tokens": 6913.7,
                "avg_output_tokens": 2379.4,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 2019.3,
                "avg_cache_write_tokens": 4891.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 44.4,
                    "median_query_seconds": 45.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 84.496,
                "latency": 21.145,
                "stderr": 1.943,
                "cost_per_test": 0.025238,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 9931.5,
                "avg_reasoning_tokens": 7885.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 21.1,
                    "median_query_seconds": 20.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-5.6-luna": {
                "accuracy": 84.391,
                "latency": 116.886,
                "stderr": 2.585,
                "cost_per_test": 0.022813,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 17992.5,
                "avg_reasoning_tokens": 15990.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 116.9,
                    "median_query_seconds": 116.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-5.2-2025-12-11": {
                "accuracy": 84.387,
                "latency": 126.205,
                "stderr": 1.856,
                "cost_per_test": 0.115422,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 7676.6,
                "avg_reasoning_tokens": 6131.7,
                "avg_cache_read_tokens": 1739.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 126.2,
                    "median_query_seconds": 125.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "thinkingmachines/inkling-small": {
                "accuracy": 84.112,
                "latency": 229.993,
                "stderr": 1.87,
                "cost_per_test": 0.019018,
                "avg_input_tokens": 6115.7,
                "avg_output_tokens": 14673.9,
                "avg_cache_read_tokens": 1771.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 230.0,
                    "median_query_seconds": 224.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "anthropic/claude-sonnet-4-5-20250929-thinking": {
                "accuracy": 84.101,
                "latency": 67.924,
                "stderr": 1.873,
                "cost_per_test": 0.082281,
                "avg_input_tokens": 6943.7,
                "avg_output_tokens": 4214.4,
                "avg_cache_read_tokens": 2019.3,
                "avg_cache_write_tokens": 4914.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 67.9,
                    "median_query_seconds": 66.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.7-flash": {
                "accuracy": 83.942,
                "latency": 16.818,
                "stderr": 2.004,
                "cost_per_test": 0.058736,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 6512.5,
                "avg_reasoning_tokens": 4501.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 16.8,
                    "median_query_seconds": 16.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "alibaba/qwen3.8-27b": {
                "accuracy": 83.849,
                "latency": 89.705,
                "stderr": 1.977,
                "cost_per_test": 0.046732,
                "avg_input_tokens": 6592.2,
                "avg_output_tokens": 14478.6,
                "avg_reasoning_tokens": 12600.2,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 89.7,
                    "median_query_seconds": 84.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "xiaomi/mimo-v2.5-pro": {
                "accuracy": 83.73,
                "latency": 90.71,
                "stderr": 2.063,
                "cost_per_test": 0.00603,
                "avg_input_tokens": 6338.2,
                "avg_output_tokens": 4228.2,
                "avg_reasoning_tokens": 2101.8,
                "avg_cache_read_tokens": 940.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 90.7,
                    "median_query_seconds": 80.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "openai/gpt-6-luna": {
                "accuracy": 83.71,
                "latency": 169.606,
                "stderr": 1.949,
                "cost_per_test": 0.009562,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 18016.2,
                "avg_reasoning_tokens": 16593.5,
                "avg_cache_read_tokens": 1826.9,
                "avg_cache_write_tokens": 4278.8,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 169.6,
                    "median_query_seconds": 163.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-5-2025-08-07": {
                "accuracy": 83.65,
                "latency": 184.536,
                "stderr": 1.936,
                "cost_per_test": 0.1015,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 9567.4,
                "avg_reasoning_tokens": 7996.1,
                "avg_cache_read_tokens": 1609.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 184.5,
                    "median_query_seconds": 194.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "tencent/hy4-preview": {
                "accuracy": 83.6,
                "latency": 349.623,
                "stderr": 2.065,
                "cost_per_test": 0.053243,
                "avg_input_tokens": 6301.8,
                "avg_output_tokens": 19667.4,
                "avg_reasoning_tokens": 17392.7,
                "avg_cache_read_tokens": 1516.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 349.6,
                    "median_query_seconds": 347.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 64000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Tencent",
                "harness": null
            },
            "zai/glm-5.2": {
                "accuracy": 83.534,
                "latency": 138.372,
                "stderr": 2.002,
                "cost_per_test": 0.044912,
                "avg_input_tokens": 6308.8,
                "avg_output_tokens": 8514.0,
                "avg_reasoning_tokens": 6588.2,
                "avg_cache_read_tokens": 1212.1,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 138.4,
                    "median_query_seconds": 127.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "anthropic/claude-opus-4-5-20251101": {
                "accuracy": 83.246,
                "latency": 43.305,
                "stderr": 1.926,
                "cost_per_test": 0.281674,
                "avg_input_tokens": 6913.7,
                "avg_output_tokens": 2027.4,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_write_tokens": 6910.7,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 43.3,
                    "median_query_seconds": 42.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-2.5-flash-thinking": {
                "accuracy": 82.983,
                "latency": 22.529,
                "stderr": 1.908,
                "cost_per_test": 0.014824,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 5151.6,
                "avg_reasoning_tokens": 3127.0,
                "avg_cache_read_tokens": 124.9,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 22.5,
                    "median_query_seconds": 20.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-opus-4-7": {
                "accuracy": 82.953,
                "latency": 67.496,
                "stderr": 1.977,
                "cost_per_test": 0.177841,
                "avg_input_tokens": 9691.5,
                "avg_output_tokens": 5139.4,
                "avg_cache_read_tokens": 1949.2,
                "avg_cache_write_tokens": 7736.3,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 67.5,
                    "median_query_seconds": 65.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-2.5-flash": {
                "accuracy": 82.869,
                "latency": 22.786,
                "stderr": 1.909,
                "cost_per_test": 0.014869,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 5239.4,
                "avg_reasoning_tokens": 3217.4,
                "avg_cache_read_tokens": 770.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 22.8,
                    "median_query_seconds": 22.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-5.6-terra": {
                "accuracy": 82.869,
                "latency": 35.526,
                "stderr": 1.948,
                "cost_per_test": 0.062588,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 4197.6,
                "avg_reasoning_tokens": 2531.2,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 35.5,
                    "median_query_seconds": 30.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-6-sol": {
                "accuracy": 82.034,
                "latency": 81.135,
                "stderr": 1.942,
                "cost_per_test": 0.083138,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 7207.0,
                "avg_reasoning_tokens": 6078.9,
                "avg_cache_read_tokens": 1826.9,
                "avg_cache_write_tokens": 4278.8,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 81.1,
                    "median_query_seconds": 74.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4-fast-reasoning": {
                "accuracy": 81.632,
                "latency": 12.423,
                "stderr": 2.137,
                "cost_per_test": 0.002535,
                "avg_input_tokens": 6133.4,
                "avg_output_tokens": 2947.1,
                "avg_reasoning_tokens": 870.9,
                "avg_cache_read_tokens": 1099.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 12.4,
                    "median_query_seconds": 12.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "ant/ling-3.0-flash-2607": {
                "accuracy": 80.904,
                "latency": 12.121,
                "stderr": 2.039,
                "cost_per_test": 0.001358,
                "avg_input_tokens": 6550.8,
                "avg_output_tokens": 4397.3,
                "avg_cache_read_tokens": 1687.5,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 12.1,
                    "median_query_seconds": 11.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Ant",
                "harness": null
            },
            "minimax/MiniMax-M2.1": {
                "accuracy": 80.777,
                "latency": 53.161,
                "stderr": 1.831,
                "cost_per_test": 0.005087,
                "avg_input_tokens": 6028.8,
                "avg_output_tokens": 2845.0,
                "avg_cache_read_tokens": 1703.9,
                "avg_cache_write_tokens": 4325.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 53.2,
                    "median_query_seconds": 54.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "MiniMax",
                "harness": null
            },
            "openai/gpt-5-mini-2025-08-07": {
                "accuracy": 80.577,
                "latency": 275.717,
                "stderr": 1.924,
                "cost_per_test": 0.033478,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 16140.4,
                "avg_reasoning_tokens": 14300.7,
                "avg_cache_read_tokens": 1468.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 275.7,
                    "median_query_seconds": 264.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "mistralai/mistral-large-4": {
                "accuracy": 80.434,
                "latency": 125.352,
                "stderr": 1.996,
                "cost_per_test": 0.023874,
                "avg_input_tokens": 6389.0,
                "avg_output_tokens": 10452.1,
                "avg_cache_read_tokens": 4043.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 125.4,
                    "median_query_seconds": 120.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 256000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Mistral AI",
                "harness": null
            },
            "deepseek/deepseek-v4-flash-0731": {
                "accuracy": 80.363,
                "latency": 77.552,
                "stderr": 1.973,
                "cost_per_test": 0.014247,
                "avg_input_tokens": 6316.4,
                "avg_output_tokens": 8687.9,
                "avg_reasoning_tokens": 7409.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 77.6,
                    "median_query_seconds": 75.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "deepseek/deepseek-v4-pro-0813": {
                "accuracy": 80.174,
                "latency": 155.859,
                "stderr": 2.004,
                "cost_per_test": 0.041127,
                "avg_input_tokens": 6329.4,
                "avg_output_tokens": 8885.2,
                "avg_reasoning_tokens": 7297.9,
                "avg_cache_read_tokens": 1891.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 155.9,
                    "median_query_seconds": 152.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "minimax/MiniMax-M2.7": {
                "accuracy": 79.867,
                "latency": 27.25,
                "stderr": 1.86,
                "cost_per_test": 0.005124,
                "avg_input_tokens": 6028.8,
                "avg_output_tokens": 2875.9,
                "avg_cache_read_tokens": 1703.9,
                "avg_cache_write_tokens": 4325.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 27.3,
                    "median_query_seconds": 27.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "MiniMax",
                "harness": null
            },
            "grok/grok-4-fast-non-reasoning": {
                "accuracy": 79.722,
                "latency": 8.482,
                "stderr": 1.871,
                "cost_per_test": 0.002056,
                "avg_input_tokens": 6145.4,
                "avg_output_tokens": 1997.9,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 1145.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 8.5,
                    "median_query_seconds": 8.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "google/gemini-3.6-flash": {
                "accuracy": 79.661,
                "latency": 33.552,
                "stderr": 1.861,
                "cost_per_test": 0.073178,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 8438.1,
                "avg_reasoning_tokens": 6614.7,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 33.6,
                    "median_query_seconds": 32.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "alibaba/qwen3.7-max": {
                "accuracy": 79.396,
                "latency": 108.046,
                "stderr": 1.907,
                "cost_per_test": 0.06907,
                "avg_input_tokens": 6554.2,
                "avg_output_tokens": 7024.6,
                "avg_reasoning_tokens": 5333.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 108.0,
                    "median_query_seconds": 103.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "grok/grok-4-1-fast-reasoning": {
                "accuracy": 78.732,
                "latency": 39.292,
                "stderr": 1.866,
                "cost_per_test": 0.002387,
                "avg_input_tokens": 6133.4,
                "avg_output_tokens": 2485.4,
                "avg_reasoning_tokens": 1071.0,
                "avg_cache_read_tokens": 552.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 39.3,
                    "median_query_seconds": 35.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "google/gemini-2.5-flash-preview-09-2025-thinking": {
                "accuracy": 78.497,
                "latency": 31.385,
                "stderr": 1.993,
                "cost_per_test": 0.014526,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 5102.2,
                "avg_reasoning_tokens": 3231.9,
                "avg_cache_read_tokens": 770.2,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 31.4,
                    "median_query_seconds": 31.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "grok/grok-4-0709": {
                "accuracy": 78.152,
                "latency": 74.054,
                "stderr": 2.084,
                "cost_per_test": 0.063955,
                "avg_input_tokens": 6662.4,
                "avg_output_tokens": 3175.4,
                "avg_reasoning_tokens": 1093.9,
                "avg_cache_read_tokens": 1628.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 74.1,
                    "median_query_seconds": 68.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "kimi/kimi-k2.6": {
                "accuracy": 78.149,
                "latency": 494.142,
                "stderr": 1.792,
                "cost_per_test": 0.055962,
                "avg_input_tokens": 6152.6,
                "avg_output_tokens": 12702.6,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 877.7,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 494.1,
                    "median_query_seconds": 478.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "google/gemini-2.5-flash-preview-09-2025": {
                "accuracy": 77.946,
                "latency": 22.893,
                "stderr": 1.923,
                "cost_per_test": 0.014385,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 4998.4,
                "avg_reasoning_tokens": 3115.2,
                "avg_cache_read_tokens": 333.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 22.9,
                    "median_query_seconds": 22.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-5.4-2026-03-05": {
                "accuracy": 77.549,
                "latency": 282.914,
                "stderr": 3.316,
                "cost_per_test": 0.639282,
                "avg_input_tokens": 6101.8,
                "avg_output_tokens": 22020.6,
                "avg_reasoning_tokens": 20444.3,
                "avg_cache_read_tokens": 1482.7,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 48,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 282.9,
                    "median_query_seconds": 276.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4-1-fast-non-reasoning": {
                "accuracy": 77.464,
                "latency": 19.865,
                "stderr": 2.04,
                "cost_per_test": 0.001782,
                "avg_input_tokens": 6145.4,
                "avg_output_tokens": 1296.2,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 634.7,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 19.9,
                    "median_query_seconds": 19.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "alibaba/qwen3-vl-plus-2025-09-23": {
                "accuracy": 77.129,
                "latency": 71.346,
                "stderr": 1.916,
                "cost_per_test": 0.02022,
                "avg_input_tokens": 6340.2,
                "avg_output_tokens": 2101.9,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 71.3,
                    "median_query_seconds": 68.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "openai/gpt-5.4-nano-2026-03-17": {
                "accuracy": 77.09,
                "latency": 20.527,
                "stderr": 1.891,
                "cost_per_test": 0.0018,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 3823.9,
                "avg_reasoning_tokens": 1905.7,
                "avg_cache_read_tokens": 768.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 20.5,
                    "median_query_seconds": 15.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "alibaba/qwen3.6-plus": {
                "accuracy": 76.963,
                "latency": 173.545,
                "stderr": 1.917,
                "cost_per_test": 0.029294,
                "avg_input_tokens": 6554.2,
                "avg_output_tokens": 8672.2,
                "avg_reasoning_tokens": 6729.1,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 173.5,
                    "median_query_seconds": 170.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "openai/o3-2025-04-16": {
                "accuracy": 76.654,
                "latency": 46.392,
                "stderr": 1.871,
                "cost_per_test": 0.040334,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 3758.4,
                "avg_reasoning_tokens": 2267.4,
                "avg_cache_read_tokens": 1300.9,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 46.4,
                    "median_query_seconds": 45.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-3.5-flash": {
                "accuracy": 76.574,
                "latency": 57.075,
                "stderr": 1.923,
                "cost_per_test": 0.166341,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 12762.6,
                "avg_reasoning_tokens": 10950.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 57.1,
                    "median_query_seconds": 57.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "kimi/kimi-k2.5-thinking": {
                "accuracy": 76.442,
                "latency": 147.155,
                "stderr": 1.986,
                "cost_per_test": 0.02489,
                "avg_input_tokens": 6152.6,
                "avg_output_tokens": 7346.5,
                "avg_cache_read_tokens": 1682.3,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 147.2,
                    "median_query_seconds": 143.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "google/gemini-3.1-pro-preview": {
                "accuracy": 76.114,
                "latency": 69.139,
                "stderr": 1.915,
                "cost_per_test": 0.097954,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 7063.7,
                "avg_reasoning_tokens": 5655.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 69.1,
                    "median_query_seconds": 68.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-sonnet-5": {
                "accuracy": 76.054,
                "latency": 251.703,
                "stderr": 3.05,
                "cost_per_test": 0.433684,
                "avg_input_tokens": 9686.5,
                "avg_output_tokens": 26809.0,
                "avg_cache_read_tokens": 1383.8,
                "avg_cache_write_tokens": 8300.7,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 251.7,
                    "median_query_seconds": 258.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-2.5-flash-lite-preview-09-2025": {
                "accuracy": 75.824,
                "latency": 4.491,
                "stderr": 1.851,
                "cost_per_test": 0.001332,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 1827.8,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 646.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 4.5,
                    "median_query_seconds": 4.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "ant/ling-3.0-flash-af-rc3": {
                "accuracy": 75.593,
                "latency": 44.811,
                "stderr": 2.026,
                "cost_per_test": 0.001618,
                "avg_input_tokens": 6550.8,
                "avg_output_tokens": 6804.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 44.8,
                    "median_query_seconds": 45.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Ant",
                "harness": null
            },
            "deepseek/deepseek-v4-pro": {
                "accuracy": 75.144,
                "latency": 345.723,
                "stderr": 2.002,
                "cost_per_test": 0.053954,
                "avg_input_tokens": 6316.4,
                "avg_output_tokens": 11557.9,
                "avg_reasoning_tokens": 10016.4,
                "avg_cache_read_tokens": 120.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 345.7,
                    "median_query_seconds": 313.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "grok/grok-4.3": {
                "accuracy": 74.399,
                "latency": 100.457,
                "stderr": 2.019,
                "cost_per_test": 0.015293,
                "avg_input_tokens": 5993.4,
                "avg_output_tokens": 5285.7,
                "avg_reasoning_tokens": 4180.5,
                "avg_cache_read_tokens": 5155.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 100.5,
                    "median_query_seconds": 62.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "anthropic/claude-opus-4-1-20250805-thinking": {
                "accuracy": 73.901,
                "latency": 57.404,
                "stderr": 1.965,
                "cost_per_test": 0.263427,
                "avg_input_tokens": 6943.7,
                "avg_output_tokens": 2241.4,
                "avg_cache_read_tokens": 2019.3,
                "avg_cache_write_tokens": 4914.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 57.4,
                    "median_query_seconds": 57.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-2.5-pro": {
                "accuracy": 73.552,
                "latency": 35.909,
                "stderr": 1.91,
                "cost_per_test": 0.046379,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 3813.5,
                "avg_reasoning_tokens": 2255.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 35.9,
                    "median_query_seconds": 34.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-5-nano-2025-08-07": {
                "accuracy": 72.865,
                "latency": 112.909,
                "stderr": 1.891,
                "cost_per_test": 0.006961,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 16804.3,
                "avg_reasoning_tokens": 15243.8,
                "avg_cache_read_tokens": 1473.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 112.9,
                    "median_query_seconds": 103.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-2.5-flash-lite": {
                "accuracy": 72.832,
                "latency": 4.847,
                "stderr": 1.982,
                "cost_per_test": 0.001211,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 1417.4,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 166.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 4.8,
                    "median_query_seconds": 4.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "alibaba/qwen3-max-2026-01-23": {
                "accuracy": 72.709,
                "latency": 362.378,
                "stderr": 1.905,
                "cost_per_test": 0.085327,
                "avg_input_tokens": 6342.2,
                "avg_output_tokens": 12952.7,
                "avg_reasoning_tokens": 11570.7,
                "avg_cache_read_tokens": 0.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 362.4,
                    "median_query_seconds": 367.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "anthropic/claude-sonnet-4-20250514": {
                "accuracy": 72.411,
                "latency": 25.674,
                "stderr": 1.929,
                "cost_per_test": 0.038973,
                "avg_input_tokens": 6913.7,
                "avg_output_tokens": 1334.3,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 2019.3,
                "avg_cache_write_tokens": 4891.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 25.7,
                    "median_query_seconds": 26.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "zai/glm-5.1": {
                "accuracy": 72.27,
                "latency": 95.701,
                "stderr": 2.064,
                "cost_per_test": 0.023717,
                "avg_input_tokens": 6301.8,
                "avg_output_tokens": 5887.8,
                "avg_reasoning_tokens": 4502.7,
                "avg_cache_read_tokens": 1781.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 95.7,
                    "median_query_seconds": 100.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "xiaomi/mimo-v2.5": {
                "accuracy": 72.151,
                "latency": 20.261,
                "stderr": 1.851,
                "cost_per_test": 0.001351,
                "avg_input_tokens": 6338.2,
                "avg_output_tokens": 2106.9,
                "avg_reasoning_tokens": 594.2,
                "avg_cache_read_tokens": 919.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 20.3,
                    "median_query_seconds": 18.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "google/gemini-3-pro-preview": {
                "accuracy": 72.036,
                "latency": 43.386,
                "stderr": 1.9,
                "cost_per_test": 0.061162,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 3997.7,
                "avg_reasoning_tokens": 2674.2,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 43.4,
                    "median_query_seconds": 41.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-opus-4-1-20250805": {
                "accuracy": 71.753,
                "latency": 38.04,
                "stderr": 2.021,
                "cost_per_test": 0.187162,
                "avg_input_tokens": 6913.7,
                "avg_output_tokens": 1231.7,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_read_tokens": 2019.3,
                "avg_cache_write_tokens": 4891.4,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 38.0,
                    "median_query_seconds": 37.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.5-flash-lite": {
                "accuracy": 70.886,
                "latency": 18.991,
                "stderr": 2.031,
                "cost_per_test": 0.019867,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 7155.4,
                "avg_reasoning_tokens": 5565.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 19.0,
                    "median_query_seconds": 18.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "alibaba/qwen3.5-flash": {
                "accuracy": 70.619,
                "latency": 80.406,
                "stderr": 2.09,
                "cost_per_test": 0.004425,
                "avg_input_tokens": 6554.2,
                "avg_output_tokens": 9424.5,
                "avg_reasoning_tokens": 7506.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 80.4,
                    "median_query_seconds": 80.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "google/gemini-3-flash-preview": {
                "accuracy": 69.917,
                "latency": 23.491,
                "stderr": 1.899,
                "cost_per_test": 0.014379,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 3693.8,
                "avg_reasoning_tokens": 1943.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 23.5,
                    "median_query_seconds": 23.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-sonnet-4-20250514-thinking": {
                "accuracy": 69.353,
                "latency": 39.572,
                "stderr": 2.212,
                "cost_per_test": 0.053443,
                "avg_input_tokens": 6952.2,
                "avg_output_tokens": 2304.4,
                "avg_cache_read_tokens": 2083.0,
                "avg_cache_write_tokens": 4859.2,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 39.6,
                    "median_query_seconds": 39.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "openai/o4-mini-2025-04-16": {
                "accuracy": 69.139,
                "latency": 81.959,
                "stderr": 1.957,
                "cost_per_test": 0.040605,
                "avg_input_tokens": 6108.7,
                "avg_output_tokens": 8004.2,
                "avg_reasoning_tokens": 6599.8,
                "avg_cache_read_tokens": 1243.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 82.0,
                    "median_query_seconds": 77.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "zai/glm-4.7": {
                "accuracy": 68.629,
                "latency": 167.479,
                "stderr": 2.123,
                "cost_per_test": 0.019082,
                "avg_input_tokens": 6301.8,
                "avg_output_tokens": 7309.8,
                "avg_reasoning_tokens": 5852.1,
                "avg_cache_read_tokens": 1593.6,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 167.5,
                    "median_query_seconds": 158.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "mistralai/mistral-medium-3.5": {
                "accuracy": 67.728,
                "latency": 104.053,
                "stderr": 2.011,
                "cost_per_test": 0.157641,
                "avg_input_tokens": 6362.4,
                "avg_output_tokens": 19746.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 104.1,
                    "median_query_seconds": 98.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Mistral AI",
                "harness": null
            },
            "google/gemini-2.5-flash-lite-preview-09-2025-thinking": {
                "accuracy": 66.877,
                "latency": 11.518,
                "stderr": 1.923,
                "cost_per_test": 0.002567,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 4914.0,
                "avg_reasoning_tokens": 3340.2,
                "avg_cache_read_tokens": 645.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 11.5,
                    "median_query_seconds": 11.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "poolside/laguna-m.1": {
                "accuracy": 65.911,
                "latency": 111.816,
                "stderr": 2.007,
                "cost_per_test": 0.002202,
                "avg_input_tokens": 6661.0,
                "avg_output_tokens": 2495.2,
                "avg_reasoning_tokens": 913.3,
                "avg_cache_read_tokens": 1286.8,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 111.8,
                    "median_query_seconds": 106.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Poolside",
                "harness": null
            },
            "google/gemini-3.1-flash-lite-preview": {
                "accuracy": 63.902,
                "latency": 16.793,
                "stderr": 1.823,
                "cost_per_test": 0.002195,
                "avg_input_tokens": 6594.9,
                "avg_output_tokens": 3839.2,
                "avg_reasoning_tokens": 2148.6,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 16.8,
                    "median_query_seconds": 12.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "grok/grok-4.20-0309-reasoning": {
                "accuracy": 63.412,
                "latency": 18.596,
                "stderr": 2.095,
                "cost_per_test": 0.031303,
                "avg_input_tokens": 6104.4,
                "avg_output_tokens": 3519.8,
                "avg_reasoning_tokens": 2680.9,
                "avg_cache_read_tokens": 1124.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 18.6,
                    "median_query_seconds": 15.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "poolside/laguna-xs.2": {
                "accuracy": 61.432,
                "latency": 68.824,
                "stderr": 2.349,
                "cost_per_test": 0.001126,
                "avg_input_tokens": 6661.0,
                "avg_output_tokens": 2732.7,
                "avg_reasoning_tokens": 968.1,
                "avg_cache_read_tokens": 1732.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 68.8,
                    "median_query_seconds": 66.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Poolside",
                "harness": null
            },
            "cohere/command-a-plus-05-2026": {
                "accuracy": 55.682,
                "latency": 203.051,
                "stderr": 3.646,
                "cost_per_test": 0.140316,
                "avg_input_tokens": 5922.7,
                "avg_output_tokens": 12550.9,
                "avg_reasoning_tokens": 11577.7,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 203.1,
                    "median_query_seconds": 157.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 64000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Cohere",
                "harness": null
            },
            "inception/mercury-2.5": {
                "accuracy": 55.093,
                "latency": 9.992,
                "stderr": 2.095,
                "cost_per_test": 0.004476,
                "avg_input_tokens": 6408.3,
                "avg_output_tokens": 4369.0,
                "avg_reasoning_tokens": 3534.2,
                "avg_cache_read_tokens": 460.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 10.0,
                    "median_query_seconds": 9.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Inception",
                "harness": null
            },
            "fireworks/llama4-maverick-instruct-basic": {
                "accuracy": 54.219,
                "latency": 25.048,
                "stderr": 1.871,
                "cost_per_test": 0.00246,
                "avg_input_tokens": 6113.7,
                "avg_output_tokens": 952.2,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 25.0,
                    "median_query_seconds": 23.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Fireworks AI",
                "harness": null
            },
            "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
                "accuracy": 50.593,
                "latency": 11.323,
                "stderr": 1.901,
                "cost_per_test": 0.0017,
                "avg_input_tokens": 6113.7,
                "avg_output_tokens": 1016.2,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 11.3,
                    "median_query_seconds": 11.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Together AI",
                "harness": null
            },
            "fireworks/nemotron-lightning-3p5-30b-a3b": {
                "accuracy": 4.267,
                "latency": 105.515,
                "stderr": 0.492,
                "cost_per_test": 0.006241,
                "avg_input_tokens": 6361.4,
                "avg_output_tokens": 29685.9,
                "avg_cache_read_tokens": 348.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 49,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 105.5,
                    "median_query_seconds": 107.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Fireworks AI",
                "harness": null
            }
        }
    }
}