{ "metadata": { "benchmark": "MedCode", "slug": "medcode", "description": "Can models support the medical billing process?", "benchmark_id": "medcode", "family": "medcode", "version": "1", "updated": "2026-10-01", "dataset_type": "private", "industry": "healthcare", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3-vl-plus-2025-09-23", "alibaba/qwen3.5-flash", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "cohere/command-a-plus-05-2026", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "fireworks/llama4-maverick-instruct-basic", "google/gemini-2.5-flash", "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-flash-thinking", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_2", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "openai/o3-2025-04-16", "openai/o4-mini-2025-04-16", "poolside/laguna-m.1", "poolside/laguna-xs.2", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-4.7", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "custom", "mode": "one-shot", "archived": false, "partner": false, "total_models": 104 }, "tasks": { "overall": { "anthropic/claude-opus-5": { "accuracy": 63.57, "latency": 24.5, "stderr": 1.993, "cost_per_test": 0.156845, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 59.062, "latency": 38.523, "stderr": 1.996, "cost_per_test": 0.024714, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 58.801, "latency": 79.527, "stderr": 2.1, "cost_per_test": 0.13716, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 56.07, "latency": 91.439, "stderr": 2.203, "cost_per_test": 0.591071, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 55.92, "latency": 44.149, "stderr": 2.112, "cost_per_test": 0.006187, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 55.825, "latency": 25.29, "stderr": 2.113, "cost_per_test": 0.073716, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 54.858, "latency": 54.251, "stderr": 2.205, "cost_per_test": 0.226314, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 53.509, "latency": 215.494, "stderr": 2.165, "cost_per_test": 1.116862, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 53.39, "latency": 9.164, "stderr": 2.12, "cost_per_test": 0.038331, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 53.217, "latency": 105.915, "stderr": 2.165, "cost_per_test": 0.350925, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 53.153, "latency": 16.685, "stderr": 2.157, "cost_per_test": 0.044216, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 52.92, "latency": 239.171, "stderr": 2.119, "cost_per_test": 0.391592, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 52.732, "latency": 54.546, "stderr": 2.151, "cost_per_test": 0.014371, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 52.198, "latency": 55.836, "stderr": 2.073, "cost_per_test": 0.028248, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark": { "accuracy": 51.31, "latency": 123.095, "stderr": 2.244, "cost_per_test": 0.005341, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 50.59, "latency": 27.415, "stderr": 2.113, "cost_per_test": 0.015389, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 49.797, "latency": 246.876, "stderr": 2.273, "cost_per_test": 0.658237, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 49.749, "latency": 156.651, "stderr": 2.262, "cost_per_test": 0.018852, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 49.634, "latency": 58.268, "stderr": 2.098, "cost_per_test": 0.045858, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 49.553, "latency": 213.135, "stderr": 2.171, "cost_per_test": 0.105497, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 49.357, "latency": 36.994, "stderr": 2.202, "cost_per_test": 0.093857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 49.346, "latency": 60.17, "stderr": 2.187, "cost_per_test": 0.039023, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 49.156, "latency": 60.839, "stderr": 2.012, "cost_per_test": 0.095846, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 49.129, "latency": 156.295, "stderr": 2.085, "cost_per_test": 0.244127, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 49.1, "latency": 159.546, "stderr": 2.188, "cost_per_test": 0.160759, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 48.839, "latency": 115.342, "stderr": 2.12, "cost_per_test": 0.084726, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 48.486, "latency": 102.865, "stderr": 2.131, "cost_per_test": 0.451358, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6": { "accuracy": 48.244, "latency": 4.058, "stderr": 2.05, "cost_per_test": 0.00618, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 48.135, "latency": 42.892, "stderr": 2.18, "cost_per_test": 0.0177, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 47.602, "latency": 8.55, "stderr": 2.071, "cost_per_test": 0.002029, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 47.54, "latency": 134.934, "stderr": 2.274, "cost_per_test": 0.278799, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 47.29, "latency": 17.676, "stderr": 2.161, "cost_per_test": 0.029818, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 47.235, "latency": 33.263, "stderr": 2.067, "cost_per_test": 0.269254, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 47.072, "latency": 62.714, "stderr": 2.119, "cost_per_test": 0.085827, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 46.289, "latency": 63.12, "stderr": 2.104, "cost_per_test": 0.012125, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 45.174, "latency": 5.023, "stderr": 1.888, "cost_per_test": 0.006826, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 44.969, "latency": 176.915, "stderr": 2.097, "cost_per_test": 0.009528, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.6": { "accuracy": 44.712, "latency": 127.284, "stderr": 2.256, "cost_per_test": 0.050335, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 44.685, "latency": 108.772, "stderr": 2.303, "cost_per_test": 0.007323, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 44.134, "latency": 74.331, "stderr": 1.998, "cost_per_test": 0.101495, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 43.974, "latency": 96.589, "stderr": 2.258, "cost_per_test": 0.280517, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 43.489, "latency": 6.019, "stderr": 1.951, "cost_per_test": 0.008093, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 43.414, "latency": 18.411, "stderr": 2.173, "cost_per_test": 0.046558, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 43.291, "latency": 59.637, "stderr": 2.313, "cost_per_test": 0.048461, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 43.247, "latency": 399.284, "stderr": 2.134, "cost_per_test": 0.062053, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 43.045, "latency": 28.173, "stderr": 2.045, "cost_per_test": 0.00556, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 42.864, "latency": 169.01, "stderr": 2.111, "cost_per_test": 0.081905, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 42.47, "latency": 180.938, "stderr": 2.16, "cost_per_test": 0.06106, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 42.391, "latency": 81.283, "stderr": 2.27, "cost_per_test": 0.01597, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 41.604, "latency": 77.578, "stderr": 2.124, "cost_per_test": 0.024196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 41.415, "latency": 127.0, "stderr": 2.15, "cost_per_test": 0.019659, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 41.372, "latency": 13.084, "stderr": 1.958, "cost_per_test": 0.20627, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 41.292, "latency": 187.265, "stderr": 2.148, "cost_per_test": 0.212108, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 41.19, "latency": 167.123, "stderr": 2.23, "cost_per_test": 0.127025, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 41.173, "latency": 35.126, "stderr": 2.042, "cost_per_test": 0.01245, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 41.057, "latency": 93.373, "stderr": 2.032, "cost_per_test": 0.002865, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 41.029, "latency": 8.043, "stderr": 2.256, "cost_per_test": 0.000844, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 40.771, "latency": 95.265, "stderr": 2.166, "cost_per_test": 0.04501, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 40.668, "latency": 378.022, "stderr": 2.029, "cost_per_test": 0.120827, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 40.569, "latency": 12.006, "stderr": 1.995, "cost_per_test": 0.042403, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 40.538, "latency": 12.702, "stderr": 1.932, "cost_per_test": 0.003692, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 40.455, "latency": 382.667, "stderr": 2.122, "cost_per_test": 0.06071, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-2.5-flash-thinking": { "accuracy": 40.357, "latency": 22.833, "stderr": 1.952, "cost_per_test": 0.003661, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 40.33, "latency": 16.106, "stderr": 1.915, "cost_per_test": 0.003653, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 40.142, "latency": 305.421, "stderr": 2.041, "cost_per_test": 0.041295, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 39.316, "latency": 75.452, "stderr": 2.119, "cost_per_test": 0.017275, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 38.751, "latency": 28.755, "stderr": 2.196, "cost_per_test": 0.042362, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 38.621, "latency": 18.675, "stderr": 2.001, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 38.425, "latency": 22.145, "stderr": 1.923, "cost_per_test": 0.003698, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 38.078, "latency": 89.002, "stderr": 2.206, "cost_per_test": 0.034103, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 38.068, "latency": 43.924, "stderr": 2.081, "cost_per_test": 0.022202, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 37.893, "latency": 191.049, "stderr": 2.206, "cost_per_test": 0.016656, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 37.385, "latency": 17.515, "stderr": 1.941, "cost_per_test": 0.002143, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 36.894, "latency": 56.222, "stderr": 2.017, "cost_per_test": 0.015673, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 36.514, "latency": 21.241, "stderr": 1.994, "cost_per_test": 0.002888, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 34.959, "latency": 39.799, "stderr": 1.939, "cost_per_test": 0.069896, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 34.44, "latency": 30.252, "stderr": 1.985, "cost_per_test": 0.007424, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 34.191, "latency": 10.49, "stderr": 1.736, "cost_per_test": 0.001182, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 34.083, "latency": 19.548, "stderr": 1.943, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 33.943, "latency": 7.298, "stderr": 1.906, "cost_per_test": 0.03946, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 33.791, "latency": 21.062, "stderr": 2.021, "cost_per_test": 0.017605, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 33.752, "latency": 34.879, "stderr": 1.148, "cost_per_test": 0.05337, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 32.997, "latency": 63.074, "stderr": 1.787, "cost_per_test": 0.003934, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.7": { "accuracy": 32.772, "latency": 123.441, "stderr": 1.996, "cost_per_test": 0.00671, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 32.678, "latency": 34.291, "stderr": 1.998, "cost_per_test": 0.020099, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 32.484, "latency": 38.448, "stderr": 1.907, "cost_per_test": 0.006719, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 32.275, "latency": 9.647, "stderr": 1.908, "cost_per_test": 0.001645, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 32.156, "latency": 16.546, "stderr": 2.124, "cost_per_test": 0.036189, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 31.895, "latency": 17.704, "stderr": 2.025, "cost_per_test": 0.002162, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3-vl-plus-2025-09-23": { "accuracy": 31.651, "latency": 9.952, "stderr": 1.845, "cost_per_test": 0.002519, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 31.373, "latency": 182.194, "stderr": 1.888, "cost_per_test": 0.014776, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "inception/mercury-2.5": { "accuracy": 31.326, "latency": 7.93, "stderr": 1.953, "cost_per_test": 0.004535, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 30.441, "latency": 29.741, "stderr": 1.948, "cost_per_test": 0.001729, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 30.036, "latency": 18.211, "stderr": 1.974, "cost_per_test": 0.002149, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 29.305, "latency": 31.167, "stderr": 1.943, "cost_per_test": 0.001354, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 28.698, "latency": 89.379, "stderr": 1.971, "cost_per_test": 0.050145, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 28.349, "latency": 4.496, "stderr": 1.921, "cost_per_test": 0.002193, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 28.08, "latency": 46.5, "stderr": 1.992, "cost_per_test": 0.002108, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite": { "accuracy": 27.115, "latency": 5.746, "stderr": 1.843, "cost_per_test": 0.001342, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 27.079, "latency": 6.239, "stderr": 1.911, "cost_per_test": 0.00144, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 23.311, "latency": 10.736, "stderr": 1.749, "cost_per_test": 0.002176, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 23.106, "latency": 70.995, "stderr": 1.693, "cost_per_test": 0.002973, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 21.251, "latency": 33.094, "stderr": 1.703, "cost_per_test": 0.001445, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "cohere/command-a-plus-05-2026": { "accuracy": 19.718, "latency": 103.751, "stderr": 1.835, "cost_per_test": 0.055695, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null } } } }