{
    "metadata": {
        "benchmark": "MedCode",
        "slug": "medcode",
        "description": "Can models support the medical billing process?",
        "benchmark_id": "medcode",
        "family": "medcode",
        "version": "1",
        "updated": "2026-10-29",
        "dataset_type": "private",
        "industry": "healthcare",
        "tasks": {
            "overall": "Overall"
        },
        "models": [
            "alibaba/qwen3-max-2026-01-23",
            "alibaba/qwen3-vl-plus-2025-09-23",
            "alibaba/qwen3.5-flash",
            "alibaba/qwen3.6-plus",
            "alibaba/qwen3.7-max",
            "alibaba/qwen3.8-27b",
            "alibaba/qwen3.8-max",
            "ant/ling-3.0-flash-2607",
            "ant/ling-3.0-flash-af-rc3",
            "anthropic/claude-fable-5",
            "anthropic/claude-fable-5-1",
            "anthropic/claude-haiku-4-5-20251001-thinking",
            "anthropic/claude-haiku-5-5",
            "anthropic/claude-opus-4-1-20250805",
            "anthropic/claude-opus-4-1-20250805-thinking",
            "anthropic/claude-opus-4-5-20251101",
            "anthropic/claude-opus-4-5-20251101-thinking",
            "anthropic/claude-opus-4-6",
            "anthropic/claude-opus-4-6-thinking",
            "anthropic/claude-opus-4-7",
            "anthropic/claude-opus-4-8",
            "anthropic/claude-opus-5",
            "anthropic/claude-opus-5-5",
            "anthropic/claude-sonnet-4-20250514",
            "anthropic/claude-sonnet-4-20250514-thinking",
            "anthropic/claude-sonnet-4-5-20250929",
            "anthropic/claude-sonnet-4-5-20250929-thinking",
            "anthropic/claude-sonnet-5",
            "anthropic/claude-sonnet-5-5",
            "cohere/command-a-plus-05-2026",
            "deepseek/deepseek-v4-flash-0731",
            "deepseek/deepseek-v4-pro",
            "deepseek/deepseek-v4-pro-0813",
            "deepseek/deepseek-v4.1-flash",
            "fireworks/llama4-maverick-instruct-basic",
            "google/gemini-2.5-flash",
            "google/gemini-2.5-flash-lite",
            "google/gemini-2.5-flash-lite-preview-09-2025",
            "google/gemini-2.5-flash-lite-preview-09-2025-thinking",
            "google/gemini-2.5-flash-preview-09-2025",
            "google/gemini-2.5-flash-preview-09-2025-thinking",
            "google/gemini-2.5-flash-thinking",
            "google/gemini-2.5-pro",
            "google/gemini-3-flash-preview",
            "google/gemini-3-pro-preview",
            "google/gemini-3.1-flash-lite-preview",
            "google/gemini-3.1-pro-preview",
            "google/gemini-3.5-flash",
            "google/gemini-3.5-flash-lite",
            "google/gemini-3.6-flash",
            "google/gemini-3.7-flash",
            "google/gemini-3.8-flash",
            "google/gemini-4-argon",
            "grok/grok-4-0709",
            "grok/grok-4-1-fast-non-reasoning",
            "grok/grok-4-1-fast-reasoning",
            "grok/grok-4-fast-non-reasoning",
            "grok/grok-4-fast-reasoning",
            "grok/grok-4.20-0309-reasoning",
            "grok/grok-4.3",
            "grok/grok-4.5",
            "grok/grok-4.6",
            "grok/grok-4.7",
            "inception/mercury-2.5",
            "kimi/kimi-k2.5-thinking",
            "kimi/kimi-k2.6",
            "kimi/kimi-k3",
            "meta/muse-wonder-full",
            "meta/muse_spark",
            "meta/muse_spark_1_2",
            "minimax/MiniMax-M2.1",
            "minimax/MiniMax-M2.7",
            "minimax/MiniMax-M3",
            "mistralai/mistral-large-4",
            "mistralai/mistral-medium-3.5",
            "nvidia/nemotron-3-ultra-550b-a55b",
            "openai/gpt-5-2025-08-07",
            "openai/gpt-5-mini-2025-08-07",
            "openai/gpt-5-nano-2025-08-07",
            "openai/gpt-5.1-2025-11-13",
            "openai/gpt-5.2-2025-12-11",
            "openai/gpt-5.4-2026-03-05",
            "openai/gpt-5.4-nano-2026-03-17",
            "openai/gpt-5.5",
            "openai/gpt-5.6-luna",
            "openai/gpt-5.6-sol",
            "openai/gpt-5.6-terra",
            "openai/gpt-6-astra",
            "openai/gpt-6-luna",
            "openai/gpt-6-sol",
            "openai/gpt-6.1-sol",
            "openai/o3-2025-04-16",
            "openai/o4-mini-2025-04-16",
            "poolside/laguna-m.1",
            "poolside/laguna-xs.2",
            "tencent/hy4-preview",
            "thinkingmachines/inkling",
            "thinkingmachines/inkling-small",
            "together/meta-llama/Llama-4-Scout-17B-16E-Instruct",
            "xiaomi/mimo-v2.5",
            "xiaomi/mimo-v2.5-pro",
            "xiaomi/mimo-v2.6-flash",
            "xiaomi/mimo-v2.6-pro",
            "zai/glm-4.7",
            "zai/glm-5.1",
            "zai/glm-5.2",
            "zai/glm-5.3"
        ],
        "partners": [],
        "showBadge": false,
        "visible": true,
        "use_cost_per_test": false,
        "runner": "custom",
        "mode": "one-shot",
        "archived": false,
        "partner": false,
        "total_models": 107
    },
    "tasks": {
        "overall": {
            "anthropic/claude-opus-5": {
                "accuracy": 63.57,
                "latency": 24.5,
                "stderr": 1.993,
                "cost_per_test": 0.156845,
                "avg_input_tokens": 17091.5,
                "avg_output_tokens": 2001.0,
                "avg_reasoning_tokens": 1308.5,
                "avg_cache_write_tokens": 17089.5,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 24.5,
                    "median_query_seconds": 21.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.1-pro-preview": {
                "accuracy": 59.062,
                "latency": 38.523,
                "stderr": 1.996,
                "cost_per_test": 0.024714,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 134.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 38.5,
                    "median_query_seconds": 35.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 58.801,
                "latency": 79.527,
                "stderr": 2.1,
                "cost_per_test": 0.13716,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 4557.6,
                "avg_reasoning_tokens": 4487.1,
                "avg_cache_read_tokens": 50.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 79.5,
                    "median_query_seconds": 62.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-fable-5": {
                "accuracy": 56.07,
                "latency": 91.439,
                "stderr": 2.203,
                "cost_per_test": 0.591071,
                "avg_input_tokens": 17004.9,
                "avg_output_tokens": 7573.6,
                "avg_cache_write_tokens": 16936.6,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 91.4,
                    "median_query_seconds": 77.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3-flash-preview": {
                "accuracy": 55.92,
                "latency": 44.149,
                "stderr": 2.112,
                "cost_per_test": 0.006187,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 137.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 44.1,
                    "median_query_seconds": 39.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "google/gemini-3.5-flash": {
                "accuracy": 55.825,
                "latency": 25.29,
                "stderr": 2.113,
                "cost_per_test": 0.073716,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 4218.0,
                "avg_reasoning_tokens": 4154.4,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 25.3,
                    "median_query_seconds": 20.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-opus-4-7": {
                "accuracy": 54.858,
                "latency": 54.251,
                "stderr": 2.205,
                "cost_per_test": 0.226314,
                "avg_input_tokens": 17097.5,
                "avg_output_tokens": 4781.8,
                "avg_cache_write_tokens": 17025.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 54.3,
                    "median_query_seconds": 32.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-fable-5-1": {
                "accuracy": 53.509,
                "latency": 215.494,
                "stderr": 2.165,
                "cost_per_test": 1.116862,
                "avg_input_tokens": 17093.5,
                "avg_output_tokens": 18064.1,
                "avg_reasoning_tokens": 17362.4,
                "avg_cache_write_tokens": 17089.5,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 215.5,
                    "median_query_seconds": 190.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.7-flash": {
                "accuracy": 53.39,
                "latency": 9.164,
                "stderr": 2.12,
                "cost_per_test": 0.038331,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 2800.9,
                "avg_reasoning_tokens": 2739.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 9.2,
                    "median_query_seconds": 8.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-opus-4-8": {
                "accuracy": 53.217,
                "latency": 105.915,
                "stderr": 2.165,
                "cost_per_test": 0.350925,
                "avg_input_tokens": 16990.9,
                "avg_output_tokens": 9792.7,
                "avg_cache_write_tokens": 16922.6,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 105.9,
                    "median_query_seconds": 80.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.6-flash": {
                "accuracy": 53.153,
                "latency": 16.685,
                "stderr": 2.157,
                "cost_per_test": 0.044216,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 3585.6,
                "avg_reasoning_tokens": 3529.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 16.7,
                    "median_query_seconds": 15.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-sonnet-5-5": {
                "accuracy": 52.92,
                "latency": 239.171,
                "stderr": 2.119,
                "cost_per_test": 0.391592,
                "avg_input_tokens": 17093.5,
                "avg_output_tokens": 34886.0,
                "avg_reasoning_tokens": 34598.9,
                "avg_cache_write_tokens": 17089.5,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 239.2,
                    "median_query_seconds": 218.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-5.1-2025-11-13": {
                "accuracy": 52.732,
                "latency": 54.546,
                "stderr": 2.151,
                "cost_per_test": 0.014371,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 80.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 54.5,
                    "median_query_seconds": 52.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-3-pro-preview": {
                "accuracy": 52.198,
                "latency": 55.836,
                "stderr": 2.073,
                "cost_per_test": 0.028248,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 429.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 55.8,
                    "median_query_seconds": 53.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-haiku-5-5": {
                "accuracy": 51.542,
                "latency": 226.464,
                "stderr": 2.166,
                "cost_per_test": 0.031937,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "meta/muse_spark": {
                "accuracy": 51.31,
                "latency": 123.095,
                "stderr": 2.244,
                "cost_per_test": 0.005341,
                "avg_input_tokens": 6181.5,
                "avg_output_tokens": 7356.2,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 123.1,
                    "median_query_seconds": 119.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "google/gemini-2.5-pro": {
                "accuracy": 50.59,
                "latency": 27.415,
                "stderr": 2.113,
                "cost_per_test": 0.015389,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 95.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 27.4,
                    "median_query_seconds": 27.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "meta/muse-wonder-full": {
                "accuracy": 49.932,
                "latency": 182.7,
                "stderr": 2.177,
                "cost_per_test": 0.247179,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "anthropic/claude-opus-5-5": {
                "accuracy": 49.797,
                "latency": 246.876,
                "stderr": 2.273,
                "cost_per_test": 0.634301,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null,
                "avg_input_tokens": 17274.7,
                "avg_output_tokens": 28593.4,
                "avg_reasoning_tokens": 27853.8,
                "avg_cache_write_tokens": 17270.7,
                "turn_metrics": {
                    "tasks_measured": 159,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 246.9,
                    "median_query_seconds": 211.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                }
            },
            "openai/gpt-5.2-2025-12-11": {
                "accuracy": 49.749,
                "latency": 156.651,
                "stderr": 2.262,
                "cost_per_test": 0.018852,
                "avg_input_tokens": 10823.7,
                "avg_output_tokens": 65.9,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 156.7,
                    "median_query_seconds": 138.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-5-2025-08-07": {
                "accuracy": 49.634,
                "latency": 58.268,
                "stderr": 2.098,
                "cost_per_test": 0.045858,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 3229.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 58.3,
                    "median_query_seconds": 54.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4.7": {
                "accuracy": 49.553,
                "latency": 213.135,
                "stderr": 2.171,
                "cost_per_test": 0.105497,
                "avg_input_tokens": 11853.8,
                "avg_output_tokens": 13963.6,
                "avg_reasoning_tokens": 13885.2,
                "avg_cache_read_tokens": 1328.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 213.1,
                    "median_query_seconds": 193.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "kimi/kimi-k3": {
                "accuracy": 49.357,
                "latency": 36.994,
                "stderr": 2.202,
                "cost_per_test": 0.093857,
                "avg_input_tokens": 11140.6,
                "avg_output_tokens": 5463.9,
                "avg_reasoning_tokens": 5247.5,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 37.0,
                    "median_query_seconds": 27.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "meta/muse_spark_1_2": {
                "accuracy": 49.346,
                "latency": 60.17,
                "stderr": 2.187,
                "cost_per_test": 0.039023,
                "avg_input_tokens": 10764.9,
                "avg_output_tokens": 6015.9,
                "avg_reasoning_tokens": 5870.6,
                "avg_cache_read_tokens": 0.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 60.2,
                    "median_query_seconds": 55.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "anthropic/claude-opus-4-5-20251101-thinking": {
                "accuracy": 49.156,
                "latency": 60.839,
                "stderr": 2.012,
                "cost_per_test": 0.095846,
                "avg_input_tokens": 655.5,
                "avg_output_tokens": 3702.7,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 60.8,
                    "median_query_seconds": 57.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "anthropic/claude-opus-4-6-thinking": {
                "accuracy": 49.129,
                "latency": 156.295,
                "stderr": 2.085,
                "cost_per_test": 0.244127,
                "avg_input_tokens": 647.2,
                "avg_output_tokens": 9635.6,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 156.3,
                    "median_query_seconds": 136.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-5.5": {
                "accuracy": 49.1,
                "latency": 159.546,
                "stderr": 2.188,
                "cost_per_test": 0.160759,
                "avg_input_tokens": 11189.2,
                "avg_output_tokens": 7256.6,
                "avg_reasoning_tokens": 7188.3,
                "avg_cache_read_tokens": 1269.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 159.5,
                    "median_query_seconds": 153.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-6.1-sol": {
                "accuracy": 48.839,
                "latency": 115.342,
                "stderr": 2.12,
                "cost_per_test": 0.084726,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 5761.0,
                "avg_reasoning_tokens": 5682.3,
                "avg_cache_write_tokens": 10821.8,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 115.3,
                    "median_query_seconds": 97.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-6-astra": {
                "accuracy": 48.486,
                "latency": 102.865,
                "stderr": 2.131,
                "cost_per_test": 0.451358,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 6856.6,
                "avg_reasoning_tokens": 6777.3,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 102.9,
                    "median_query_seconds": 94.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-opus-4-6": {
                "accuracy": 48.244,
                "latency": 4.058,
                "stderr": 2.05,
                "cost_per_test": 0.00618,
                "avg_input_tokens": 692.4,
                "avg_output_tokens": 108.7,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 4.1,
                    "median_query_seconds": 3.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 48.135,
                "latency": 42.892,
                "stderr": 2.18,
                "cost_per_test": 0.0177,
                "avg_input_tokens": 11518.0,
                "avg_output_tokens": 16067.4,
                "avg_reasoning_tokens": 16010.9,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 42.9,
                    "median_query_seconds": 38.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "google/gemini-3.1-flash-lite-preview": {
                "accuracy": 47.602,
                "latency": 8.55,
                "stderr": 2.071,
                "cost_per_test": 0.002029,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 2186.2,
                "avg_reasoning_tokens": 2076.9,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 8.6,
                    "median_query_seconds": 6.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "anthropic/claude-sonnet-5": {
                "accuracy": 47.54,
                "latency": 134.934,
                "stderr": 2.274,
                "cost_per_test": 0.278799,
                "avg_input_tokens": 16776.4,
                "avg_output_tokens": 14421.2,
                "avg_reasoning_tokens": 13704.5,
                "avg_cache_read_tokens": 121.8,
                "avg_cache_write_tokens": 16641.5,
                "turn_metrics": {
                    "tasks_measured": 158,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 134.9,
                    "median_query_seconds": 121.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "openai/o3-2025-04-16": {
                "accuracy": 47.29,
                "latency": 17.676,
                "stderr": 2.161,
                "cost_per_test": 0.029818,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 1014.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 17.7,
                    "median_query_seconds": 17.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-opus-4-1-20250805-thinking": {
                "accuracy": 47.235,
                "latency": 33.263,
                "stderr": 2.067,
                "cost_per_test": 0.269254,
                "avg_input_tokens": 12346.3,
                "avg_output_tokens": 1120.8,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 33.3,
                    "median_query_seconds": 32.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-6-sol": {
                "accuracy": 47.072,
                "latency": 62.714,
                "stderr": 2.119,
                "cost_per_test": 0.085827,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 5871.1,
                "avg_reasoning_tokens": 5804.5,
                "avg_cache_write_tokens": 10821.8,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 62.7,
                    "median_query_seconds": 59.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "minimax/MiniMax-M3": {
                "accuracy": 46.289,
                "latency": 63.12,
                "stderr": 2.104,
                "cost_per_test": 0.012125,
                "avg_input_tokens": 10761.1,
                "avg_output_tokens": 2384.6,
                "avg_cache_read_tokens": 114.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 63.1,
                    "median_query_seconds": 44.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "MiniMax",
                "harness": null
            },
            "anthropic/claude-opus-4-5-20251101": {
                "accuracy": 45.174,
                "latency": 5.023,
                "stderr": 1.888,
                "cost_per_test": 0.006826,
                "avg_input_tokens": 692.4,
                "avg_output_tokens": 134.6,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 5.0,
                    "median_query_seconds": 4.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "high",
                "provider": "Anthropic",
                "harness": null
            },
            "xiaomi/mimo-v2.6-pro": {
                "accuracy": 44.969,
                "latency": 176.915,
                "stderr": 2.097,
                "cost_per_test": 0.009528,
                "avg_input_tokens": 12032.9,
                "avg_output_tokens": 8459.8,
                "avg_reasoning_tokens": 8303.9,
                "avg_cache_read_tokens": 7108.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 176.9,
                    "median_query_seconds": 84.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 44.712,
                "latency": 127.284,
                "stderr": 2.256,
                "cost_per_test": 0.050335,
                "avg_input_tokens": 10641.8,
                "avg_output_tokens": 4873.0,
                "avg_reasoning_tokens": 4816.7,
                "avg_cache_read_tokens": 124.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 127.3,
                    "median_query_seconds": 118.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "openai/gpt-6-luna": {
                "accuracy": 44.685,
                "latency": 108.772,
                "stderr": 2.303,
                "cost_per_test": 0.007323,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 11935.2,
                "avg_reasoning_tokens": 11876.4,
                "avg_cache_write_tokens": 10821.8,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 108.8,
                    "median_query_seconds": 85.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "anthropic/claude-sonnet-4-5-20250929-thinking": {
                "accuracy": 44.134,
                "latency": 74.331,
                "stderr": 1.998,
                "cost_per_test": 0.101495,
                "avg_input_tokens": 12512.0,
                "avg_output_tokens": 4263.9,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 74.3,
                    "median_query_seconds": 78.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-5.6-sol": {
                "accuracy": 43.974,
                "latency": 96.589,
                "stderr": 2.258,
                "cost_per_test": 0.280517,
                "avg_input_tokens": 11341.4,
                "avg_output_tokens": 7460.3,
                "avg_reasoning_tokens": 7402.2,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 96.6,
                    "median_query_seconds": 67.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "google/gemini-3.5-flash-lite": {
                "accuracy": 43.489,
                "latency": 6.019,
                "stderr": 1.951,
                "cost_per_test": 0.008093,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 1851.2,
                "avg_reasoning_tokens": 1798.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 6.0,
                    "median_query_seconds": 5.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-5.6-terra": {
                "accuracy": 43.414,
                "latency": 18.411,
                "stderr": 2.173,
                "cost_per_test": 0.046558,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 2071.0,
                "avg_reasoning_tokens": 2008.4,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 18.4,
                    "median_query_seconds": 11.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4.5": {
                "accuracy": 43.291,
                "latency": 59.637,
                "stderr": 2.313,
                "cost_per_test": 0.048461,
                "avg_input_tokens": 10641.8,
                "avg_output_tokens": 4529.6,
                "avg_reasoning_tokens": 4476.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 59.6,
                    "median_query_seconds": 55.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "tencent/hy4-preview": {
                "accuracy": 43.247,
                "latency": 399.284,
                "stderr": 2.134,
                "cost_per_test": 0.062053,
                "avg_input_tokens": 10851.3,
                "avg_output_tokens": 21471.5,
                "avg_reasoning_tokens": 21404.0,
                "avg_cache_read_tokens": 880.9,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 399.3,
                    "median_query_seconds": 391.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 64000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Tencent",
                "harness": null
            },
            "openai/gpt-5-mini-2025-08-07": {
                "accuracy": 43.045,
                "latency": 28.173,
                "stderr": 2.045,
                "cost_per_test": 0.00556,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 1423.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 28.2,
                    "median_query_seconds": 25.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "zai/glm-5.3": {
                "accuracy": 42.864,
                "latency": 169.01,
                "stderr": 2.111,
                "cost_per_test": 0.081905,
                "avg_input_tokens": 10253.1,
                "avg_output_tokens": 15399.1,
                "avg_cache_read_tokens": 179.9,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 169.0,
                    "median_query_seconds": 161.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "deepseek/deepseek-v4-pro-0813": {
                "accuracy": 42.47,
                "latency": 180.938,
                "stderr": 2.16,
                "cost_per_test": 0.06106,
                "avg_input_tokens": 10815.1,
                "avg_output_tokens": 11814.1,
                "avg_reasoning_tokens": 11756.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 180.9,
                    "median_query_seconds": 177.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "openai/gpt-5.6-luna": {
                "accuracy": 42.391,
                "latency": 81.283,
                "stderr": 2.27,
                "cost_per_test": 0.01597,
                "avg_input_tokens": 10740.1,
                "avg_output_tokens": 11518.3,
                "avg_reasoning_tokens": 11459.7,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 81.3,
                    "median_query_seconds": 110.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "zai/glm-5.1": {
                "accuracy": 41.604,
                "latency": 77.578,
                "stderr": 2.124,
                "cost_per_test": 0.024196,
                "avg_input_tokens": 11115.3,
                "avg_output_tokens": 4177.0,
                "avg_reasoning_tokens": 4108.0,
                "avg_cache_read_tokens": 357.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 77.6,
                    "median_query_seconds": 72.2,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "deepseek/deepseek-v4-flash-0731": {
                "accuracy": 41.415,
                "latency": 127.0,
                "stderr": 2.15,
                "cost_per_test": 0.019659,
                "avg_input_tokens": 10760.1,
                "avg_output_tokens": 14691.8,
                "avg_reasoning_tokens": 14632.1,
                "avg_cache_read_tokens": 10489.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 127.0,
                    "median_query_seconds": 128.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "anthropic/claude-opus-4-1-20250805": {
                "accuracy": 41.372,
                "latency": 13.084,
                "stderr": 1.958,
                "cost_per_test": 0.20627,
                "avg_input_tokens": 12318.3,
                "avg_output_tokens": 286.6,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 13.1,
                    "median_query_seconds": 12.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "openai/gpt-5.4-2026-03-05": {
                "accuracy": 41.292,
                "latency": 187.265,
                "stderr": 2.148,
                "cost_per_test": 0.212108,
                "avg_input_tokens": 10983.9,
                "avg_output_tokens": 12309.9,
                "avg_reasoning_tokens": 12256.7,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 187.3,
                    "median_query_seconds": 193.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "thinkingmachines/inkling": {
                "accuracy": 41.19,
                "latency": 167.123,
                "stderr": 2.23,
                "cost_per_test": 0.127025,
                "avg_input_tokens": 10860.6,
                "avg_output_tokens": 9253.0,
                "avg_cache_read_tokens": 67.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 167.1,
                    "median_query_seconds": 146.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "deepseek/deepseek-v4.1-flash": {
                "accuracy": 41.173,
                "latency": 35.126,
                "stderr": 2.042,
                "cost_per_test": 0.01245,
                "avg_input_tokens": 10749.7,
                "avg_output_tokens": 7687.8,
                "avg_reasoning_tokens": 7633.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 35.1,
                    "median_query_seconds": 31.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 384000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "xiaomi/mimo-v2.6-flash": {
                "accuracy": 41.057,
                "latency": 93.373,
                "stderr": 2.032,
                "cost_per_test": 0.002865,
                "avg_input_tokens": 12032.9,
                "avg_output_tokens": 4658.3,
                "avg_reasoning_tokens": 4490.1,
                "avg_cache_read_tokens": 901.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 93.4,
                    "median_query_seconds": 37.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "openai/gpt-5.4-nano-2026-03-17": {
                "accuracy": 41.029,
                "latency": 8.043,
                "stderr": 2.256,
                "cost_per_test": 0.000844,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 754.0,
                "avg_reasoning_tokens": 693.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 8.0,
                    "median_query_seconds": 7.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "zai/glm-5.2": {
                "accuracy": 40.771,
                "latency": 95.265,
                "stderr": 2.166,
                "cost_per_test": 0.04501,
                "avg_input_tokens": 11121.9,
                "avg_output_tokens": 6690.9,
                "avg_reasoning_tokens": 6615.4,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 95.3,
                    "median_query_seconds": 66.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "mistralai/mistral-large-4": {
                "accuracy": 40.685,
                "latency": 68.682,
                "stderr": 2.171,
                "cost_per_test": 0.021241,
                "avg_input_tokens": 11124.5,
                "avg_output_tokens": 6406.7,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 68.7,
                    "median_query_seconds": 67.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 256000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Mistral AI",
                "harness": null
            },
            "alibaba/qwen3.8-max": {
                "accuracy": 40.668,
                "latency": 378.022,
                "stderr": 2.029,
                "cost_per_test": 0.120827,
                "avg_input_tokens": 11940.4,
                "avg_output_tokens": 16390.8,
                "avg_reasoning_tokens": 16314.8,
                "avg_cache_read_tokens": 799.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 378.0,
                    "median_query_seconds": 376.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "anthropic/claude-sonnet-4-5-20250929": {
                "accuracy": 40.569,
                "latency": 12.006,
                "stderr": 1.995,
                "cost_per_test": 0.042403,
                "avg_input_tokens": 12484.0,
                "avg_output_tokens": 330.1,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 12.0,
                    "median_query_seconds": 11.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "google/gemini-2.5-flash-preview-09-2025": {
                "accuracy": 40.538,
                "latency": 12.702,
                "stderr": 1.932,
                "cost_per_test": 0.003692,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 90.8,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 12.7,
                    "median_query_seconds": 11.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "deepseek/deepseek-v4-pro": {
                "accuracy": 40.455,
                "latency": 382.667,
                "stderr": 2.122,
                "cost_per_test": 0.06071,
                "avg_input_tokens": 10802.7,
                "avg_output_tokens": 11800.4,
                "avg_reasoning_tokens": 11741.3,
                "avg_cache_read_tokens": 218.8,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 382.7,
                    "median_query_seconds": 363.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "google/gemini-2.5-flash-thinking": {
                "accuracy": 40.357,
                "latency": 22.833,
                "stderr": 1.952,
                "cost_per_test": 0.003661,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 78.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 22.8,
                    "median_query_seconds": 21.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "google/gemini-2.5-flash-preview-09-2025-thinking": {
                "accuracy": 40.33,
                "latency": 16.106,
                "stderr": 1.915,
                "cost_per_test": 0.003653,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 75.1,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 16.1,
                    "median_query_seconds": 14.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "kimi/kimi-k2.6": {
                "accuracy": 40.142,
                "latency": 305.421,
                "stderr": 2.041,
                "cost_per_test": 0.041295,
                "avg_input_tokens": 10984.2,
                "avg_output_tokens": 7715.1,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 164,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 305.4,
                    "median_query_seconds": 273.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "kimi/kimi-k2.5-thinking": {
                "accuracy": 39.316,
                "latency": 75.452,
                "stderr": 2.119,
                "cost_per_test": 0.017275,
                "avg_input_tokens": 11062.6,
                "avg_output_tokens": 3546.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 75.5,
                    "median_query_seconds": 76.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "alibaba/qwen3.7-max": {
                "accuracy": 38.751,
                "latency": 28.755,
                "stderr": 2.196,
                "cost_per_test": 0.042362,
                "avg_input_tokens": 11986.1,
                "avg_output_tokens": 1652.9,
                "avg_reasoning_tokens": 1575.6,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 28.8,
                    "median_query_seconds": 27.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "nvidia/nemotron-3-ultra-550b-a55b": {
                "accuracy": 38.621,
                "latency": 18.675,
                "stderr": 2.001,
                "cost_per_test": null,
                "avg_input_tokens": 12065.2,
                "avg_output_tokens": 2101.7,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 18.7,
                    "median_query_seconds": 17.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Nvidia",
                "harness": null
            },
            "google/gemini-2.5-flash": {
                "accuracy": 38.425,
                "latency": 22.145,
                "stderr": 1.923,
                "cost_per_test": 0.003698,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 93.0,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 22.1,
                    "median_query_seconds": 19.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "grok/grok-4-0709": {
                "accuracy": 38.078,
                "latency": 89.002,
                "stderr": 2.206,
                "cost_per_test": 0.034103,
                "avg_input_tokens": 11124.5,
                "avg_output_tokens": 48.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 89.0,
                    "median_query_seconds": 76.0,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "grok/grok-4.3": {
                "accuracy": 38.068,
                "latency": 43.924,
                "stderr": 2.081,
                "cost_per_test": 0.022202,
                "avg_input_tokens": 10453.5,
                "avg_output_tokens": 3654.2,
                "avg_reasoning_tokens": 3608.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 43.9,
                    "median_query_seconds": 43.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "thinkingmachines/inkling-small": {
                "accuracy": 37.893,
                "latency": 191.049,
                "stderr": 2.206,
                "cost_per_test": 0.016656,
                "avg_input_tokens": 10860.6,
                "avg_output_tokens": 11186.4,
                "avg_cache_read_tokens": 106.3,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 191.0,
                    "median_query_seconds": 196.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "0.99",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Thinkingmachines",
                "harness": null
            },
            "grok/grok-4-fast-reasoning": {
                "accuracy": 37.385,
                "latency": 17.515,
                "stderr": 1.941,
                "cost_per_test": 0.002143,
                "avg_input_tokens": 10596.5,
                "avg_output_tokens": 46.9,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 17.5,
                    "median_query_seconds": 14.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "alibaba/qwen3.6-plus": {
                "accuracy": 36.894,
                "latency": 56.222,
                "stderr": 2.017,
                "cost_per_test": 0.015673,
                "avg_input_tokens": 11986.1,
                "avg_output_tokens": 3226.8,
                "avg_reasoning_tokens": 3155.2,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 56.2,
                    "median_query_seconds": 56.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "fireworks/llama4-maverick-instruct-basic": {
                "accuracy": 36.514,
                "latency": 21.241,
                "stderr": 1.994,
                "cost_per_test": 0.002888,
                "avg_input_tokens": 10767.9,
                "avg_output_tokens": 589.9,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 21.2,
                    "median_query_seconds": 19.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Fireworks AI",
                "harness": null
            },
            "anthropic/claude-sonnet-4-20250514-thinking": {
                "accuracy": 34.959,
                "latency": 39.799,
                "stderr": 1.939,
                "cost_per_test": 0.069896,
                "avg_input_tokens": 12121.7,
                "avg_output_tokens": 2235.4,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 39.8,
                    "median_query_seconds": 33.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "minimax/MiniMax-M2.7": {
                "accuracy": 34.44,
                "latency": 30.252,
                "stderr": 1.985,
                "cost_per_test": 0.007424,
                "avg_input_tokens": 10588.0,
                "avg_output_tokens": 2882.1,
                "avg_cache_read_tokens": 15.9,
                "avg_cache_write_tokens": 10572.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 30.3,
                    "median_query_seconds": 15.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "MiniMax",
                "harness": null
            },
            "google/gemini-2.5-flash-lite-preview-09-2025-thinking": {
                "accuracy": 34.191,
                "latency": 10.49,
                "stderr": 1.736,
                "cost_per_test": 0.001182,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 67.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 10.5,
                    "median_query_seconds": 10.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "minimax/MiniMax-M2.1": {
                "accuracy": 34.083,
                "latency": 19.548,
                "stderr": 1.943,
                "cost_per_test": null,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 19.5,
                    "median_query_seconds": 15.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "MiniMax",
                "harness": null
            },
            "anthropic/claude-sonnet-4-20250514": {
                "accuracy": 33.943,
                "latency": 7.298,
                "stderr": 1.906,
                "cost_per_test": 0.03946,
                "avg_input_tokens": 12093.7,
                "avg_output_tokens": 212.0,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 7.3,
                    "median_query_seconds": 6.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "openai/o4-mini-2025-04-16": {
                "accuracy": 33.791,
                "latency": 21.062,
                "stderr": 2.021,
                "cost_per_test": 0.017605,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 1288.1,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 21.1,
                    "median_query_seconds": 19.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "mistralai/mistral-medium-3.5": {
                "accuracy": 33.752,
                "latency": 34.879,
                "stderr": 1.148,
                "cost_per_test": 0.05337,
                "avg_input_tokens": 10990.1,
                "avg_output_tokens": 4918.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 164,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 34.9,
                    "median_query_seconds": 32.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Mistral AI",
                "harness": null
            },
            "alibaba/qwen3.5-flash": {
                "accuracy": 32.997,
                "latency": 63.074,
                "stderr": 1.787,
                "cost_per_test": 0.003934,
                "avg_input_tokens": 11987.1,
                "avg_output_tokens": 6839.2,
                "avg_reasoning_tokens": 6766.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 63.1,
                    "median_query_seconds": 63.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "zai/glm-4.7": {
                "accuracy": 32.772,
                "latency": 123.441,
                "stderr": 1.996,
                "cost_per_test": 0.00671,
                "avg_input_tokens": 10966.4,
                "avg_output_tokens": 59.1,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 123.4,
                    "median_query_seconds": 105.8,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "anthropic/claude-haiku-4-5-20251001-thinking": {
                "accuracy": 32.678,
                "latency": 34.291,
                "stderr": 1.998,
                "cost_per_test": 0.020099,
                "avg_input_tokens": 655.5,
                "avg_output_tokens": 3888.6,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 34.3,
                    "median_query_seconds": 33.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Anthropic",
                "harness": null
            },
            "xiaomi/mimo-v2.5-pro": {
                "accuracy": 32.484,
                "latency": 38.448,
                "stderr": 1.907,
                "cost_per_test": 0.006719,
                "avg_input_tokens": 12276.9,
                "avg_output_tokens": 1679.8,
                "avg_reasoning_tokens": 1609.8,
                "avg_cache_read_tokens": 192.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 38.4,
                    "median_query_seconds": 32.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "ant/ling-3.0-flash-2607": {
                "accuracy": 32.275,
                "latency": 9.647,
                "stderr": 1.908,
                "cost_per_test": 0.001645,
                "avg_input_tokens": 11867.9,
                "avg_output_tokens": 3430.6,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 9.6,
                    "median_query_seconds": 7.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Ant",
                "harness": null
            },
            "grok/grok-4.20-0309-reasoning": {
                "accuracy": 32.156,
                "latency": 16.546,
                "stderr": 2.124,
                "cost_per_test": 0.036189,
                "avg_input_tokens": 10566.4,
                "avg_output_tokens": 2571.6,
                "avg_reasoning_tokens": 2531.2,
                "avg_cache_read_tokens": 207.5,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 16.5,
                    "median_query_seconds": 14.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "xiaomi/mimo-v2.5": {
                "accuracy": 31.895,
                "latency": 17.704,
                "stderr": 2.025,
                "cost_per_test": 0.002162,
                "avg_input_tokens": 12272.9,
                "avg_output_tokens": 1680.0,
                "avg_reasoning_tokens": 1614.9,
                "avg_cache_read_tokens": 192.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 17.7,
                    "median_query_seconds": 16.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "alibaba/qwen3-vl-plus-2025-09-23": {
                "accuracy": 31.651,
                "latency": 9.952,
                "stderr": 1.845,
                "cost_per_test": 0.002519,
                "avg_input_tokens": 12033.9,
                "avg_output_tokens": 69.9,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 10.0,
                    "median_query_seconds": 7.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "alibaba/qwen3-max-2026-01-23": {
                "accuracy": 31.373,
                "latency": 182.194,
                "stderr": 1.888,
                "cost_per_test": 0.014776,
                "avg_input_tokens": 12035.9,
                "avg_output_tokens": 55.4,
                "avg_cache_read_tokens": 0.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 182.2,
                    "median_query_seconds": 173.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "inception/mercury-2.5": {
                "accuracy": 31.326,
                "latency": 7.93,
                "stderr": 1.953,
                "cost_per_test": 0.004535,
                "avg_input_tokens": 11669.7,
                "avg_output_tokens": 2934.7,
                "avg_reasoning_tokens": 2872.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 7.9,
                    "median_query_seconds": 6.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Inception",
                "harness": null
            },
            "openai/gpt-5-nano-2025-08-07": {
                "accuracy": 30.441,
                "latency": 29.741,
                "stderr": 1.948,
                "cost_per_test": 0.001729,
                "avg_input_tokens": 10852.6,
                "avg_output_tokens": 2965.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 29.7,
                    "median_query_seconds": 29.3,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "grok/grok-4-fast-non-reasoning": {
                "accuracy": 30.036,
                "latency": 18.211,
                "stderr": 1.974,
                "cost_per_test": 0.002149,
                "avg_input_tokens": 10608.5,
                "avg_output_tokens": 53.7,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 18.2,
                    "median_query_seconds": 13.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "ant/ling-3.0-flash-af-rc3": {
                "accuracy": 29.305,
                "latency": 31.167,
                "stderr": 1.943,
                "cost_per_test": 0.001354,
                "avg_input_tokens": 11867.9,
                "avg_output_tokens": 3566.2,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 31.2,
                    "median_query_seconds": 29.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Ant",
                "harness": null
            },
            "alibaba/qwen3.8-27b": {
                "accuracy": 28.698,
                "latency": 89.379,
                "stderr": 1.971,
                "cost_per_test": 0.050145,
                "avg_input_tokens": 12003.4,
                "avg_output_tokens": 14714.4,
                "avg_reasoning_tokens": 14641.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 89.4,
                    "median_query_seconds": 89.5,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "grok/grok-4-1-fast-non-reasoning": {
                "accuracy": 28.349,
                "latency": 4.496,
                "stderr": 1.921,
                "cost_per_test": 0.002193,
                "avg_input_tokens": 10441.8,
                "avg_output_tokens": 209.5,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 4.5,
                    "median_query_seconds": 4.4,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "grok/grok-4-1-fast-reasoning": {
                "accuracy": 28.08,
                "latency": 46.5,
                "stderr": 1.992,
                "cost_per_test": 0.002108,
                "avg_input_tokens": 10440.4,
                "avg_output_tokens": 39.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 46.5,
                    "median_query_seconds": 41.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "google/gemini-2.5-flash-lite": {
                "accuracy": 27.115,
                "latency": 5.746,
                "stderr": 1.843,
                "cost_per_test": 0.001342,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 467.4,
                "avg_reasoning_tokens": 0.0,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 5.7,
                    "median_query_seconds": 4.9,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "google/gemini-2.5-flash-lite-preview-09-2025": {
                "accuracy": 27.079,
                "latency": 6.239,
                "stderr": 1.911,
                "cost_per_test": 0.00144,
                "avg_input_tokens": 11549.8,
                "avg_output_tokens": 712.1,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 6.2,
                    "median_query_seconds": 6.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
                "accuracy": 23.311,
                "latency": 10.736,
                "stderr": 1.749,
                "cost_per_test": 0.002176,
                "avg_input_tokens": 10767.9,
                "avg_output_tokens": 403.3,
                "avg_reasoning_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 10.7,
                    "median_query_seconds": 9.7,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Together AI",
                "harness": null
            },
            "poolside/laguna-m.1": {
                "accuracy": 23.106,
                "latency": 70.995,
                "stderr": 1.693,
                "cost_per_test": 0.002973,
                "avg_input_tokens": 12371.4,
                "avg_output_tokens": 1251.3,
                "avg_reasoning_tokens": 1188.3,
                "avg_cache_read_tokens": 17.1,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 71.0,
                    "median_query_seconds": 54.6,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Poolside",
                "harness": null
            },
            "poolside/laguna-xs.2": {
                "accuracy": 21.251,
                "latency": 33.094,
                "stderr": 1.703,
                "cost_per_test": 0.001445,
                "avg_input_tokens": 12408.4,
                "avg_output_tokens": 1030.5,
                "avg_reasoning_tokens": 960.4,
                "avg_cache_read_tokens": 48.6,
                "avg_cache_write_tokens": 0.0,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 33.1,
                    "median_query_seconds": 31.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 30000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Poolside",
                "harness": null
            },
            "cohere/command-a-plus-05-2026": {
                "accuracy": 19.718,
                "latency": 103.751,
                "stderr": 1.835,
                "cost_per_test": 0.055695,
                "avg_input_tokens": 9567.2,
                "avg_output_tokens": 3490.9,
                "avg_reasoning_tokens": 3403.7,
                "turn_metrics": {
                    "tasks_measured": 165,
                    "avg_turns": 1.0,
                    "avg_query_seconds": 103.8,
                    "median_query_seconds": 62.1,
                    "avg_ttft_seconds": null,
                    "output_time_split": null
                },
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 64000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Cohere",
                "harness": null
            }
        }
    }
}
