{
  "metadata": {
    "benchmark": "Vals Router Index",
    "slug": "vals_router_index",
    "description": "How model routers compare with single models on long agentic work from the Vals Index.",
    "benchmark_id": "vals_router_index",
    "family": "vals_router_index",
    "version": "1.0",
    "updated": "2026-11-03",
    "dataset_type": "private",
    "industry": "index",
    "tasks": {
      "overall": "Overall",
      "finance_agent": "Finance Agent",
      "emb": "EMB",
      "terminal_bench_4": "Terminal-Bench 4.0",
      "vibe_code_bench": "Vibe Code Bench",
      "code_migration": "Code Migration",
      "legal_research": "Legal Research",
      "legal_agent_benchmark": "HLAB",
      "tax_agent_bench": "Tax Agent Bench"
    },
    "models": [
      "anthropic/claude-haiku-5-5",
      "anthropic/claude-opus-5-5",
      "anthropic/claude-sonnet-5-5",
      "deepseek/deepseek-v4-pro-0813",
      "deepseek/deepseek-v4.1-flash-1017",
      "fireworks/firerouter",
      "fireworks/firerouter-sol",
      "google/gemini-3.8-flash",
      "google/gemini-4-argon",
      "grok/grok-4.7",
      "kimi/kimi-k3",
      "meta/muse-wonder",
      "meta/muse-wonder-full",
      "minimax/MiniMax-M3.1-flash",
      "openai/gpt-6-astra",
      "openai/gpt-6-luna",
      "openai/gpt-6.1-sol",
      "openrouter/openrouter/auto",
      "openrouter/typesafe/jev-router",
      "ramp/switchyard",
      "sakana/fugu-max-v1.0",
      "tencent/hy4-preview",
      "thinkingmachines/inkling",
      "thinkingmachines/inkling-small",
      "xiaomi/mimo-v2.6-flash",
      "xiaomi/mimo-v2.6-pro",
      "zai/glm-5.3",
      "zai/glm-5.3-flash"
    ],
    "partners": [],
    "showBadge": false,
    "visible": true,
    "use_cost_per_test": true,
    "runner": "external",
    "mode": "agentic",
    "archived": false,
    "partner": false,
    "total_models": 28,
    "router_chart": true
  },
  "tasks": {
    "overall": {
      "openrouter/openrouter/auto": {
        "accuracy": 52.195,
        "latency": 2420.133,
        "stderr": 1.207,
        "cost_per_test": 0.817334,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 56.458,
        "latency": 2807.558,
        "stderr": 1.303,
        "cost_per_test": 9.217354,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 60.019,
        "latency": 912.615,
        "stderr": 1.285,
        "cost_per_test": 1.253046,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 57.795,
        "latency": 680.404,
        "stderr": 1.167,
        "cost_per_test": 0.956238,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 55.652,
        "latency": 1478.241,
        "stderr": 1.332,
        "cost_per_test": 5.303181,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 48.114,
        "latency": 850.32,
        "stderr": 1.166,
        "cost_per_test": 3.912391,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 58.708,
        "latency": 2988.332,
        "stderr": 1.282,
        "cost_per_test": 3.173012,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 64.156,
        "latency": 5071.089,
        "stderr": 1.369,
        "cost_per_test": 24.408239,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 69.028,
        "latency": 5158.766,
        "stderr": 1.084,
        "cost_per_test": 15.874117,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 52.974,
        "latency": 3482.913,
        "stderr": 1.044,
        "cost_per_test": 2.499691,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 61.257,
        "latency": 2394.709,
        "stderr": 1.055,
        "cost_per_test": 0.65747,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 55.951,
        "latency": 2158.482,
        "stderr": 1.141,
        "cost_per_test": 6.064799,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 69.237,
        "latency": 2697.953,
        "stderr": 1.09,
        "cost_per_test": 13.372731,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 58.564,
        "latency": 2013.837,
        "stderr": 1.013,
        "cost_per_test": 11.625468,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 52.955,
        "latency": 4496.011,
        "stderr": 1.001,
        "cost_per_test": 7.169465,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 64.877,
        "latency": 2123.575,
        "stderr": 1.067,
        "cost_per_test": 10.647504,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 66.787,
        "latency": 3558.574,
        "stderr": 1.066,
        "cost_per_test": 6.809627,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 53.781,
        "latency": 2597.835,
        "stderr": 1.067,
        "cost_per_test": 0.659684,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 65.023,
        "latency": 2018.879,
        "stderr": 1.098,
        "cost_per_test": 21.277828,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 52.322,
        "latency": 1943.192,
        "stderr": 1.117,
        "cost_per_test": 0.450147,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 63.944,
        "latency": 2664.03,
        "stderr": 1.072,
        "cost_per_test": 3.614054,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 53.209,
        "latency": 3562.637,
        "stderr": 1.13,
        "cost_per_test": 1.446335,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 28.961,
        "latency": 1032.925,
        "stderr": 1.12,
        "cost_per_test": 1.847767,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 27.588,
        "latency": 986.441,
        "stderr": 1.133,
        "cost_per_test": 0.401307,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 54.826,
        "latency": 4027.975,
        "stderr": 1.114,
        "cost_per_test": 0.221566,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 57.813,
        "latency": 4355.892,
        "stderr": 1.24,
        "cost_per_test": 0.434692,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 59.729,
        "latency": 3864.087,
        "stderr": 1.111,
        "cost_per_test": 8.524625,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 50.861,
        "latency": 3830.634,
        "stderr": 1.255,
        "cost_per_test": 2.048666,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    },
    "finance_agent": {
      "openrouter/openrouter/auto": {
        "accuracy": 59.098,
        "latency": 268.433,
        "stderr": 2.201,
        "cost_per_test": 0.09512,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 64.456,
        "latency": 147.97,
        "stderr": 2.14,
        "cost_per_test": 0.653802,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 62.293,
        "latency": 200.793,
        "stderr": 2.155,
        "cost_per_test": 0.36199,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 62.451,
        "latency": 127.076,
        "stderr": 2.145,
        "cost_per_test": 0.145253,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 59.614,
        "latency": 261.498,
        "stderr": 2.183,
        "cost_per_test": 0.895878,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 54.99,
        "latency": 177.443,
        "stderr": 2.217,
        "cost_per_test": 0.952185,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 65.373,
        "latency": 1864.634,
        "stderr": 2.063,
        "cost_per_test": 1.637243,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 66.761,
        "latency": 1980.122,
        "stderr": 2.118,
        "cost_per_test": 8.818176,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 61.704,
        "latency": 2066.221,
        "stderr": 2.186,
        "cost_per_test": 6.464159,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 57.116,
        "latency": 476.985,
        "stderr": 2.083,
        "cost_per_test": 0.82234,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 62.933,
        "latency": 909.212,
        "stderr": 2.028,
        "cost_per_test": 0.466261,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 65.761,
        "latency": 443.888,
        "stderr": 2.039,
        "cost_per_test": 1.991077,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 67.658,
        "latency": 793.03,
        "stderr": 2.0,
        "cost_per_test": 4.315759,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 60.585,
        "latency": 652.292,
        "stderr": 2.114,
        "cost_per_test": 2.238652,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 60.013,
        "latency": 875.039,
        "stderr": 2.09,
        "cost_per_test": 1.881233,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 65.545,
        "latency": 310.129,
        "stderr": 2.031,
        "cost_per_test": 2.620705,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 67.575,
        "latency": 850.298,
        "stderr": 1.98,
        "cost_per_test": 2.680843,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 58.826,
        "latency": 904.204,
        "stderr": 2.07,
        "cost_per_test": 0.158978,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 60.428,
        "latency": 1228.75,
        "stderr": 2.12,
        "cost_per_test": 7.823464,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 56.841,
        "latency": 926.859,
        "stderr": 2.123,
        "cost_per_test": 0.120919,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 61.817,
        "latency": 1145.078,
        "stderr": 2.112,
        "cost_per_test": 1.626543,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 61.623,
        "latency": 1502.748,
        "stderr": 2.059,
        "cost_per_test": 0.667449,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 49.873,
        "latency": 573.747,
        "stderr": 2.116,
        "cost_per_test": 0.891309,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 44.776,
        "latency": 314.216,
        "stderr": 2.095,
        "cost_per_test": 0.207921,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 63.316,
        "latency": 936.347,
        "stderr": 2.084,
        "cost_per_test": 0.066534,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 64.259,
        "latency": 682.01,
        "stderr": 2.015,
        "cost_per_test": 0.183321,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 65.004,
        "latency": 1205.572,
        "stderr": 2.01,
        "cost_per_test": 1.030939,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 60.587,
        "latency": 296.095,
        "stderr": 2.032,
        "cost_per_test": 0.072473,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    },
    "emb": {
      "openrouter/openrouter/auto": {
        "accuracy": 67.174,
        "latency": 902.021,
        "stderr": 1.95,
        "cost_per_test": 0.204939,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 73.016,
        "latency": 731.624,
        "stderr": 1.895,
        "cost_per_test": 2.002952,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 71.836,
        "latency": 320.124,
        "stderr": 2.01,
        "cost_per_test": 0.282921,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 69.61,
        "latency": 354.757,
        "stderr": 2.223,
        "cost_per_test": 0.378797,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 70.977,
        "latency": 551.08,
        "stderr": 2.23,
        "cost_per_test": 2.029949,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 66.927,
        "latency": 460.153,
        "stderr": 2.213,
        "cost_per_test": 0.854517,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 65.477,
        "latency": 1195.527,
        "stderr": 3.691,
        "cost_per_test": 1.364559,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 75.836,
        "latency": 2106.434,
        "stderr": 2.097,
        "cost_per_test": 9.330415,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 75.899,
        "latency": 2204.991,
        "stderr": 2.1,
        "cost_per_test": 6.526758,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 65.128,
        "latency": 2247.236,
        "stderr": 2.35,
        "cost_per_test": 1.08141,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 69.173,
        "latency": 1971.687,
        "stderr": 2.395,
        "cost_per_test": 0.330922,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 72.871,
        "latency": 769.197,
        "stderr": 2.424,
        "cost_per_test": 8.243862,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 74.828,
        "latency": 2305.821,
        "stderr": 2.494,
        "cost_per_test": 11.8029,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 71.498,
        "latency": 2097.023,
        "stderr": 2.328,
        "cost_per_test": 6.481284,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 71.264,
        "latency": 741.42,
        "stderr": 2.283,
        "cost_per_test": 3.074655,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 72.503,
        "latency": 1034.437,
        "stderr": 2.414,
        "cost_per_test": 5.599337,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 74.998,
        "latency": 1684.504,
        "stderr": 2.141,
        "cost_per_test": 4.58223,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 67.843,
        "latency": 1847.78,
        "stderr": 2.29,
        "cost_per_test": 0.268958,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 72.683,
        "latency": 804.28,
        "stderr": 2.297,
        "cost_per_test": 5.827063,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 69.954,
        "latency": 1041.052,
        "stderr": 2.542,
        "cost_per_test": 0.141708,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 71.088,
        "latency": 1107.811,
        "stderr": 2.748,
        "cost_per_test": 1.143199,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 68.946,
        "latency": 1839.41,
        "stderr": 2.402,
        "cost_per_test": 0.787396,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 40.971,
        "latency": 1581.835,
        "stderr": 2.982,
        "cost_per_test": 1.671783,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 40.823,
        "latency": 1173.625,
        "stderr": 3.21,
        "cost_per_test": 0.249301,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 68.882,
        "latency": 1521.906,
        "stderr": 2.267,
        "cost_per_test": 0.108634,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 67.347,
        "latency": 3313.924,
        "stderr": 2.903,
        "cost_per_test": 0.328999,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 69.112,
        "latency": 2172.231,
        "stderr": 2.649,
        "cost_per_test": 3.787827,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 66.252,
        "latency": 4495.202,
        "stderr": 2.685,
        "cost_per_test": 1.426066,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    },
    "terminal_bench_4": {
      "openrouter/openrouter/auto": {
        "accuracy": 25.758,
        "latency": 2604.973,
        "stderr": 4.058,
        "cost_per_test": 0.862076,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 46.465,
        "latency": 3465.113,
        "stderr": 5.38,
        "cost_per_test": 7.274694,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 45.96,
        "latency": 1122.296,
        "stderr": 5.077,
        "cost_per_test": 1.161956,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 32.323,
        "latency": 1184.575,
        "stderr": 4.772,
        "cost_per_test": 1.089203,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 45.96,
        "latency": 1353.451,
        "stderr": 5.178,
        "cost_per_test": 2.375726,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 17.172,
        "latency": 1901.851,
        "stderr": 3.616,
        "cost_per_test": 8.364969,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 36.364,
        "latency": 3116.385,
        "stderr": 2.314,
        "cost_per_test": 3.117464,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 58.081,
        "latency": 3810.852,
        "stderr": 5.353,
        "cost_per_test": 10.94185,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 62.626,
        "latency": 4928.927,
        "stderr": 5.166,
        "cost_per_test": 10.619851,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 14.141,
        "latency": 5493.603,
        "stderr": 2.02,
        "cost_per_test": 3.311817,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 31.818,
        "latency": 2470.449,
        "stderr": 2.624,
        "cost_per_test": 0.539359,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 19.697,
        "latency": 6427.575,
        "stderr": 2.314,
        "cost_per_test": 8.64749,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 57.576,
        "latency": 3214.793,
        "stderr": 2.314,
        "cost_per_test": 17.63994,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 28.788,
        "latency": 3007.523,
        "stderr": 2.314,
        "cost_per_test": 18.011582,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 17.172,
        "latency": 11353.52,
        "stderr": 0.505,
        "cost_per_test": 12.478176,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 49.495,
        "latency": 3280.687,
        "stderr": 2.201,
        "cost_per_test": 14.657694,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 58.081,
        "latency": 4330.665,
        "stderr": 3.312,
        "cost_per_test": 10.528007,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 25.253,
        "latency": 3030.551,
        "stderr": 1.01,
        "cost_per_test": 0.734722,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 59.596,
        "latency": 2142.082,
        "stderr": 4.403,
        "cost_per_test": 9.584259,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 13.636,
        "latency": 1965.568,
        "stderr": 1.515,
        "cost_per_test": 0.354726,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 55.051,
        "latency": 2711.667,
        "stderr": 1.821,
        "cost_per_test": 1.72496,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 8.081,
        "latency": 7798.686,
        "stderr": 1.336,
        "cost_per_test": 2.09742,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 0.505,
        "latency": 1378.754,
        "stderr": 0.505,
        "cost_per_test": 2.495998,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 0.0,
        "latency": 705.115,
        "stderr": 0.0,
        "cost_per_test": 0.321679,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 24.242,
        "latency": 8712.24,
        "stderr": 1.515,
        "cost_per_test": 0.216275,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 31.313,
        "latency": 9720.048,
        "stderr": 3.072,
        "cost_per_test": 0.497505,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 38.889,
        "latency": 4952.947,
        "stderr": 1.821,
        "cost_per_test": 9.37436,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 37.374,
        "latency": 3681.073,
        "stderr": 2.672,
        "cost_per_test": 1.219907,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    },
    "vibe_code_bench": {
      "openrouter/openrouter/auto": {
        "accuracy": 72.257,
        "latency": 2642.737,
        "stderr": 4.666,
        "cost_per_test": 1.002777,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 78.786,
        "latency": 1205.389,
        "stderr": 4.649,
        "cost_per_test": 6.51642,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 74.665,
        "latency": 2078.26,
        "stderr": 5.152,
        "cost_per_test": 4.240584,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 86.475,
        "latency": 1309.887,
        "stderr": 2.124,
        "cost_per_test": 3.16861,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 61.391,
        "latency": 856.124,
        "stderr": 5.535,
        "cost_per_test": 4.551245,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 69.608,
        "latency": 1750.313,
        "stderr": 4.14,
        "cost_per_test": 15.437519,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 90.444,
        "latency": 2075.723,
        "stderr": 1.725,
        "cost_per_test": 6.071856,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 83.339,
        "latency": 5766.524,
        "stderr": 3.789,
        "cost_per_test": 34.962712,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 92.392,
        "latency": 3672.106,
        "stderr": 1.26,
        "cost_per_test": 19.729112,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 82.297,
        "latency": 3982.6,
        "stderr": 3.148,
        "cost_per_test": 0.356023,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 87.73,
        "latency": 973.834,
        "stderr": 2.752,
        "cost_per_test": 0.579984,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 78.651,
        "latency": 519.104,
        "stderr": 3.884,
        "cost_per_test": 6.865141,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 91.906,
        "latency": 2247.675,
        "stderr": 1.899,
        "cost_per_test": 8.10797,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 86.175,
        "latency": 2147.065,
        "stderr": 2.181,
        "cost_per_test": 15.82706,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 84.968,
        "latency": 999.021,
        "stderr": 2.746,
        "cost_per_test": 10.007576,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 83.089,
        "latency": 350.076,
        "stderr": 2.49,
        "cost_per_test": 1.802723,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 85.996,
        "latency": 648.336,
        "stderr": 1.933,
        "cost_per_test": 2.293448,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 73.944,
        "latency": 1263.916,
        "stderr": 4.249,
        "cost_per_test": 1.146649,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 89.594,
        "latency": 2344.127,
        "stderr": 2.174,
        "cost_per_test": 38.512323,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 81.649,
        "latency": 2219.515,
        "stderr": 3.377,
        "cost_per_test": 1.345703,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 88.931,
        "latency": 4489.755,
        "stderr": 1.893,
        "cost_per_test": 6.234794,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 77.478,
        "latency": 2493.104,
        "stderr": 4.035,
        "cost_per_test": 2.181516,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 19.214,
        "latency": 1105.492,
        "stderr": 3.659,
        "cost_per_test": 1.605949,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 19.052,
        "latency": 2949.469,
        "stderr": 3.403,
        "cost_per_test": 0.432264,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 78.96,
        "latency": 3013.012,
        "stderr": 4.045,
        "cost_per_test": 0.563041,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 85.223,
        "latency": 3646.574,
        "stderr": 3.392,
        "cost_per_test": 1.043846,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 78.125,
        "latency": 3847.197,
        "stderr": 3.94,
        "cost_per_test": 12.450185,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 30.759,
        "latency": 5027.281,
        "stderr": 5.277,
        "cost_per_test": 4.69761,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    },
    "code_migration": {
      "openrouter/openrouter/auto": {
        "accuracy": 44.815,
        "latency": 10049.686,
        "stderr": 4.266,
        "cost_per_test": 3.736539,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 21.25,
        "latency": 14502.675,
        "stderr": 4.871,
        "cost_per_test": 47.473857,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 63.729,
        "latency": 2304.561,
        "stderr": 4.062,
        "cost_per_test": 2.044263,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 52.61,
        "latency": 1451.158,
        "stderr": 4.261,
        "cost_per_test": 1.215162,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 54.683,
        "latency": 6605.332,
        "stderr": 3.918,
        "cost_per_test": 26.506681,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 33.946,
        "latency": 1391.26,
        "stderr": 3.816,
        "cost_per_test": 2.702462,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 50.199,
        "latency": 5148.212,
        "stderr": 4.208,
        "cost_per_test": 6.130933,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 47.485,
        "latency": 12635.272,
        "stderr": 6.273,
        "cost_per_test": 72.30083,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 87.372,
        "latency": 14850.395,
        "stderr": 2.388,
        "cost_per_test": 45.405088,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 52.028,
        "latency": 10883.534,
        "stderr": 3.359,
        "cost_per_test": 11.806673,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 68.5,
        "latency": 8133.069,
        "stderr": 3.476,
        "cost_per_test": 1.572628,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 38.173,
        "latency": 7443.728,
        "stderr": 4.282,
        "cost_per_test": 16.6269,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 74.429,
        "latency": 8400.232,
        "stderr": 4.512,
        "cost_per_test": 44.644196,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 51.821,
        "latency": 3787.561,
        "stderr": 3.201,
        "cost_per_test": 32.981557,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 22.669,
        "latency": 16267.502,
        "stderr": 3.551,
        "cost_per_test": 19.510208,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 69.781,
        "latency": 9687.179,
        "stderr": 4.035,
        "cost_per_test": 47.552905,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 61.652,
        "latency": 15566.738,
        "stderr": 4.371,
        "cost_per_test": 23.236252,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 48.195,
        "latency": 7363.872,
        "stderr": 3.35,
        "cost_per_test": 1.729595,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 78.031,
        "latency": 5286.504,
        "stderr": 3.128,
        "cost_per_test": 66.641779,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 45.451,
        "latency": 3543.575,
        "stderr": 4.218,
        "cost_per_test": 0.708851,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 75.136,
        "latency": 6157.942,
        "stderr": 3.609,
        "cost_per_test": 9.938473,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 44.695,
        "latency": 6574.27,
        "stderr": 4.376,
        "cost_per_test": 3.421469,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 8.57,
        "latency": 1413.991,
        "stderr": 2.746,
        "cost_per_test": 6.017274,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 10.398,
        "latency": 1133.764,
        "stderr": 2.373,
        "cost_per_test": 1.587286,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 38.764,
        "latency": 14739.113,
        "stderr": 4.306,
        "cost_per_test": 0.612992,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 47.073,
        "latency": 12258.712,
        "stderr": 4.707,
        "cost_per_test": 0.891745,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 54.075,
        "latency": 10660.633,
        "stderr": 3.209,
        "cost_per_test": 33.523556,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 50.625,
        "latency": 12036.086,
        "stderr": 3.999,
        "cost_per_test": 8.271572,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    },
    "legal_research": {
      "openrouter/openrouter/auto": {
        "accuracy": 29.808,
        "latency": 1808.769,
        "stderr": 3.179,
        "cost_per_test": 0.207931,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 41.346,
        "latency": 383.829,
        "stderr": 3.423,
        "cost_per_test": 0.727174,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 37.5,
        "latency": 370.33,
        "stderr": 3.365,
        "cost_per_test": 0.389553,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 36.538,
        "latency": 236.513,
        "stderr": 3.347,
        "cost_per_test": 0.258068,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 29.808,
        "latency": 415.838,
        "stderr": 3.179,
        "cost_per_test": 0.902983,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 24.519,
        "latency": 373.239,
        "stderr": 2.99,
        "cost_per_test": 0.713539,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 43.269,
        "latency": 4911.734,
        "stderr": 3.444,
        "cost_per_test": 1.633459,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 50.481,
        "latency": 6749.531,
        "stderr": 3.475,
        "cost_per_test": 23.188254,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 44.231,
        "latency": 5588.989,
        "stderr": 3.452,
        "cost_per_test": 13.281914,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 40.865,
        "latency": 2527.177,
        "stderr": 3.417,
        "cost_per_test": 1.088017,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 45.192,
        "latency": 2106.881,
        "stderr": 3.459,
        "cost_per_test": 0.69856,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 38.942,
        "latency": 324.318,
        "stderr": 3.389,
        "cost_per_test": 1.630498,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 54.808,
        "latency": 1881.045,
        "stderr": 3.459,
        "cost_per_test": 6.44908,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 47.115,
        "latency": 1274.337,
        "stderr": 3.469,
        "cost_per_test": 4.920812,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 46.154,
        "latency": 760.122,
        "stderr": 3.465,
        "cost_per_test": 3.469601,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 50.481,
        "latency": 927.111,
        "stderr": 3.475,
        "cost_per_test": 3.980242,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 53.365,
        "latency": 1654.378,
        "stderr": 3.467,
        "cost_per_test": 3.595401,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 34.135,
        "latency": 2911.983,
        "stderr": 3.296,
        "cost_per_test": 0.444777,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 39.423,
        "latency": 1491.339,
        "stderr": 3.397,
        "cost_per_test": 10.577143,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 30.288,
        "latency": 2547.747,
        "stderr": 3.194,
        "cost_per_test": 0.440296,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 38.462,
        "latency": 2060.48,
        "stderr": 3.381,
        "cost_per_test": 3.384561,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 45.192,
        "latency": 3839.511,
        "stderr": 3.459,
        "cost_per_test": 0.846372,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 28.365,
        "latency": 869.377,
        "stderr": 3.133,
        "cost_per_test": 0.494044,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 25.481,
        "latency": 774.823,
        "stderr": 3.029,
        "cost_per_test": 0.141548,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 37.981,
        "latency": 919.689,
        "stderr": 3.373,
        "cost_per_test": 0.070424,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 47.115,
        "latency": 1816.322,
        "stderr": 3.469,
        "cost_per_test": 0.181201,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 49.038,
        "latency": 3110.171,
        "stderr": 3.475,
        "cost_per_test": 2.241388,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 45.192,
        "latency": 3067.486,
        "stderr": 3.459,
        "cost_per_test": 0.240392,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    },
    "legal_agent_benchmark": {
      "openrouter/openrouter/auto": {
        "accuracy": 5.0,
        "latency": 713.15,
        "stderr": 1.814,
        "cost_per_test": 0.395779,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 6.25,
        "latency": 1850.28,
        "stderr": 2.012,
        "cost_per_test": 8.739914,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 5.833,
        "latency": 672.271,
        "stderr": 1.888,
        "cost_per_test": 1.274452,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 5.0,
        "latency": 629.575,
        "stderr": 1.908,
        "cost_per_test": 1.251771,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 1.25,
        "latency": 1466.576,
        "stderr": 0.929,
        "cost_per_test": 4.470621,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 2.917,
        "latency": 486.197,
        "stderr": 1.226,
        "cost_per_test": 1.779858,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 4.167,
        "latency": 2620.795,
        "stderr": 1.518,
        "cost_per_test": 4.264376,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 9.167,
        "latency": 3507.428,
        "stderr": 2.366,
        "cost_per_test": 21.37949,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 10.417,
        "latency": 4397.953,
        "stderr": 2.572,
        "cost_per_test": 16.612583,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 10.833,
        "latency": 655.413,
        "stderr": 2.723,
        "cost_per_test": 0.802111,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 14.167,
        "latency": 1328.108,
        "stderr": 2.787,
        "cost_per_test": 0.612596,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 12.083,
        "latency": 1161.715,
        "stderr": 2.579,
        "cost_per_test": 3.656207,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 27.083,
        "latency": 1755.397,
        "stderr": 3.738,
        "cost_per_test": 10.54836,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 18.333,
        "latency": 2148.428,
        "stderr": 3.183,
        "cost_per_test": 10.746259,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 12.917,
        "latency": 2556.302,
        "stderr": 2.807,
        "cost_per_test": 4.144144,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 23.75,
        "latency": 906.14,
        "stderr": 3.646,
        "cost_per_test": 7.081987,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 23.333,
        "latency": 2649.375,
        "stderr": 3.547,
        "cost_per_test": 5.413598,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 7.5,
        "latency": 1497.601,
        "stderr": 2.265,
        "cost_per_test": 0.472848,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 7.083,
        "latency": 1953.362,
        "stderr": 1.899,
        "cost_per_test": 25.460295,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 4.167,
        "latency": 1305.488,
        "stderr": 1.629,
        "cost_per_test": 0.298018,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 6.25,
        "latency": 2425.625,
        "stderr": 1.83,
        "cost_per_test": 3.536302,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 8.75,
        "latency": 1989.335,
        "stderr": 2.187,
        "cost_per_test": 0.98378,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 3.75,
        "latency": 529.644,
        "stderr": 1.469,
        "cost_per_test": 1.252023,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 2.917,
        "latency": 455.265,
        "stderr": 1.362,
        "cost_per_test": 0.17869,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 13.333,
        "latency": 1342.563,
        "stderr": 2.821,
        "cost_per_test": 0.085646,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 13.75,
        "latency": 1861.848,
        "stderr": 2.836,
        "cost_per_test": 0.216614,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 14.583,
        "latency": 1911.01,
        "stderr": 2.862,
        "cost_per_test": 3.987774,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 10.417,
        "latency": 744.843,
        "stderr": 2.572,
        "cost_per_test": 0.323243,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    },
    "tax_agent_bench": {
      "openrouter/openrouter/auto": {
        "accuracy": 11.996,
        "latency": 371.294,
        "stderr": 2.308,
        "cost_per_test": 0.03351,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter": {
        "accuracy": 23.796,
        "latency": 173.584,
        "stderr": 3.07,
        "cost_per_test": 0.350018,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "fireworks/firerouter-sol": {
        "accuracy": 23.297,
        "latency": 232.281,
        "stderr": 3.049,
        "cost_per_test": 0.268648,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "ramp/switchyard": {
        "accuracy": 22.28,
        "latency": 149.691,
        "stderr": 3.003,
        "cost_per_test": 0.143038,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openrouter/typesafe/jev-router": {
        "accuracy": 16.036,
        "latency": 316.031,
        "stderr": 2.646,
        "cost_per_test": 0.692363,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "sakana/fugu-max-v1.0": {
        "accuracy": 13.316,
        "latency": 262.102,
        "stderr": 2.44,
        "cost_per_test": 0.494077,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-haiku-5-5": {
        "accuracy": 31.922,
        "latency": 2973.647,
        "stderr": 3.352,
        "cost_per_test": 1.164207,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-opus-5-5": {
        "accuracy": 45.573,
        "latency": 4012.548,
        "stderr": 3.593,
        "cost_per_test": 14.344183,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "anthropic/claude-sonnet-5-5": {
        "accuracy": 40.922,
        "latency": 3560.542,
        "stderr": 3.548,
        "cost_per_test": 8.353469,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4-pro-0813": {
        "accuracy": 26.756,
        "latency": 1596.755,
        "stderr": 3.093,
        "cost_per_test": 0.729136,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "deepseek/deepseek-v4.1-flash-1017": {
        "accuracy": 44.35,
        "latency": 1264.434,
        "stderr": 3.574,
        "cost_per_test": 0.459454,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-3.8-flash": {
        "accuracy": 32.124,
        "latency": 178.329,
        "stderr": 3.37,
        "cost_per_test": 0.857214,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "google/gemini-4-argon": {
        "accuracy": 44.56,
        "latency": 985.634,
        "stderr": 3.587,
        "cost_per_test": 3.473645,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "grok/grok-4.7": {
        "accuracy": 34.197,
        "latency": 996.469,
        "stderr": 3.423,
        "cost_per_test": 1.796539,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "kimi/kimi-k3": {
        "accuracy": 33.368,
        "latency": 2415.159,
        "stderr": 3.386,
        "cost_per_test": 2.790127,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder": {
        "accuracy": 36.269,
        "latency": 492.843,
        "stderr": 3.47,
        "cost_per_test": 1.884439,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "meta/muse-wonder-full": {
        "accuracy": 42.997,
        "latency": 1084.296,
        "stderr": 3.572,
        "cost_per_test": 2.147235,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "minimax/MiniMax-M3.1-flash": {
        "accuracy": 32.136,
        "latency": 1962.771,
        "stderr": 3.342,
        "cost_per_test": 0.320948,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-astra": {
        "accuracy": 20.717,
        "latency": 900.586,
        "stderr": 2.924,
        "cost_per_test": 5.7963,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6-luna": {
        "accuracy": 19.947,
        "latency": 1995.736,
        "stderr": 2.866,
        "cost_per_test": 0.190956,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "openai/gpt-6.1-sol": {
        "accuracy": 21.754,
        "latency": 1213.878,
        "stderr": 2.977,
        "cost_per_test": 1.323601,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "tencent/hy4-preview": {
        "accuracy": 32.617,
        "latency": 2464.028,
        "stderr": 3.381,
        "cost_per_test": 0.585277,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling": {
        "accuracy": 15.291,
        "latency": 810.557,
        "stderr": 2.423,
        "cost_per_test": 0.353757,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "thinkingmachines/inkling-small": {
        "accuracy": 15.389,
        "latency": 385.252,
        "stderr": 2.365,
        "cost_per_test": 0.091767,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-flash": {
        "accuracy": 28.507,
        "latency": 1038.928,
        "stderr": 3.14,
        "cost_per_test": 0.048982,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "xiaomi/mimo-v2.6-pro": {
        "accuracy": 32.388,
        "latency": 1547.695,
        "stderr": 3.361,
        "cost_per_test": 0.134306,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3": {
        "accuracy": 39.698,
        "latency": 3052.932,
        "stderr": 3.52,
        "cost_per_test": 1.80097,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      },
      "zai/glm-5.3-flash": {
        "accuracy": 31.55,
        "latency": 1297.006,
        "stderr": 3.35,
        "cost_per_test": 0.138062,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": null,
        "provider": null,
        "harness": null
      }
    }
  }
}
