{ "metadata": { "benchmark": "MortgageTax", "slug": "mortgage_tax", "description": "Evaluating reading and understanding tax certificates as images", "benchmark_id": "mortgage_tax", "family": "mortgage_tax", "version": "1", "updated": "2026-09-01", "dataset_type": "private", "industry": "finance", "tasks": { "overall": "Overall", "semantic_extraction": "Semantic Extraction", "numerical_extraction": "Numerical Extraction" }, "models": [ "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "fireworks/llama4-maverick-instruct-basic", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.0-flash-exp", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-04-17", "google/gemini-2.5-flash-preview-04-17-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemma-4-31b-it", "grok/grok-2-vision-1212", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2503", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o3-2025-04-16", "openai/o4-mini-2025-04-16", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo", "together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "xiaomi/mimo-v2.5" ], "partners": [ { "name": "vontive", "url": "https://www.vontive.com/" }, { "name": "tai", "url": "https://www.together.ai/" } ], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": false, "partner": false, "total_models": 98 }, "tasks": { "overall": { "anthropic/claude-opus-5": { "accuracy": 72.059, "latency": 7.978, "stderr": 0.875, "cost_per_test": 0.07375, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 70.786, "latency": 14.98, "stderr": 0.896, "cost_per_test": 0.171632, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 70.27, "latency": 16.101, "stderr": 0.896, "cost_per_test": 0.084417, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 70.032, "latency": 28.668, "stderr": 0.897, "cost_per_test": 0.080204, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 69.912, "latency": 20.814, "stderr": 0.893, "cost_per_test": 0.105627, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 69.396, "latency": 23.616, "stderr": 0.908, "cost_per_test": 0.023542, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 69.078, "latency": 25.602, "stderr": 0.911, "cost_per_test": 0.041198, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 68.919, "latency": 16.096, "stderr": 0.909, "cost_per_test": 0.175013, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 68.918, "latency": 24.008, "stderr": 0.907, "cost_per_test": 0.004794, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 68.76, "latency": 28.243, "stderr": 0.912, "cost_per_test": 0.071407, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 68.72, "latency": 12.838, "stderr": 0.914, "cost_per_test": 0.007477, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 68.68, "latency": 4.879, "stderr": 0.914, "cost_per_test": 0.004323, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 68.68, "latency": 5.67, "stderr": 1.775, "cost_per_test": 0.015796, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 68.68, "latency": 10.534, "stderr": 0.908, "cost_per_test": 0.092448, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 68.522, "latency": 27.161, "stderr": 0.911, "cost_per_test": 0.060749, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 68.362, "latency": 26.756, "stderr": 0.91, "cost_per_test": 0.008326, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 68.323, "latency": 45.002, "stderr": 0.915, "cost_per_test": 0.103445, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 68.283, "latency": 42.074, "stderr": 0.916, "cost_per_test": 0.016906, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 68.124, "latency": 11.428, "stderr": 0.91, "cost_per_test": 0.017873, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 68.044, "latency": 5.368, "stderr": 0.912, "cost_per_test": 0.00284, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 67.965, "latency": 78.18, "stderr": 0.908, "cost_per_test": 0.013476, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 67.726, "latency": 29.74, "stderr": 0.921, "cost_per_test": 0.04467, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 67.686, "latency": 28.166, "stderr": 0.917, "cost_per_test": 0.178388, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 67.608, "latency": 9.239, "stderr": 0.915, "cost_per_test": 0.019658, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 67.368, "latency": 36.822, "stderr": 0.922, "cost_per_test": 0.002158, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 67.329, "latency": 6.316, "stderr": 0.925, "cost_per_test": 0.022706, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 67.29, "latency": 22.835, "stderr": 0.921, "cost_per_test": 0.004804, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.289, "latency": 12.618, "stderr": 0.922, "cost_per_test": 0.071198, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 67.17, "latency": 8.91, "stderr": 0.924, "cost_per_test": 0.003638, "temperature": 0.1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 67.13, "latency": 70.666, "stderr": 0.918, "cost_per_test": 0.050478, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 66.892, "latency": 24.864, "stderr": 0.923, "cost_per_test": 0.00436, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 66.852, "latency": 13.585, "stderr": 1.804, "cost_per_test": 0.02365, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 66.654, "latency": 4.114, "stderr": 0.923, "cost_per_test": 0.011071, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 66.534, "latency": 79.787, "stderr": 0.925, "cost_per_test": 0.014918, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.336, "latency": 78.943, "stderr": 0.909, "cost_per_test": 0.038981, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 66.176, "latency": 60.548, "stderr": 0.928, "cost_per_test": 0.007588, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 66.136, "latency": 24.818, "stderr": 0.928, "cost_per_test": 0.022312, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 65.938, "latency": 5.034, "stderr": 0.922, "cost_per_test": 0.006212, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 65.818, "latency": 109.76, "stderr": 0.93, "cost_per_test": 0.02869, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 65.7, "latency": 21.216, "stderr": 0.933, "cost_per_test": 0.009972, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 65.501, "latency": 4.598, "stderr": 0.938, "cost_per_test": 0.002797, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 65.454, "latency": 62.828, "stderr": 0.879, "cost_per_test": 0.028906, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 65.422, "latency": 44.623, "stderr": 0.934, "cost_per_test": 0.026128, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 65.342, "latency": 14.794, "stderr": 0.943, "cost_per_test": 0.046973, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 64.944, "latency": 79.952, "stderr": 0.947, "cost_per_test": 0.011604, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 64.826, "latency": 12.996, "stderr": 0.944, "cost_per_test": 0.011654, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 64.189, "latency": 8.548, "stderr": 0.952, "cost_per_test": 0.013712, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 64.07, "latency": 4.224, "stderr": 1.805, "cost_per_test": 0.015076, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 63.99, "latency": 47.499, "stderr": 0.942, "cost_per_test": 0.027349, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 63.99, "latency": 47.622, "stderr": 0.955, "cost_per_test": 0.052096, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 63.99, "latency": 69.108, "stderr": 0.944, "cost_per_test": 0.056492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 63.514, "latency": 126.454, "stderr": 0.906, "cost_per_test": 0.044769, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 62.599, "latency": 7.274, "stderr": 0.956, "cost_per_test": 0.001152, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 62.468, "latency": 8.422, "stderr": 0.957, "cost_per_test": 0.015974, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 62.282, "latency": 112.214, "stderr": 0.952, "cost_per_test": 0.011526, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 62.162, "latency": 29.792, "stderr": 0.962, "cost_per_test": 0.020742, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 62.122, "latency": 10.648, "stderr": 0.952, "cost_per_test": 0.004484, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 61.924, "latency": 18.331, "stderr": 0.958, "cost_per_test": 0.001154, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 61.804, "latency": 117.461, "stderr": 0.968, "cost_per_test": 0.014998, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 61.368, "latency": 41.928, "stderr": 0.967, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 61.368, "latency": 46.254, "stderr": 0.931, "cost_per_test": 0.026408, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 61.208, "latency": 8.313, "stderr": 0.956, "cost_per_test": 0.000488, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 61.089, "latency": 10.065, "stderr": 0.969, "cost_per_test": 0.088232, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 60.97, "latency": 7.356, "stderr": 1.862, "cost_per_test": 0.006956, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 60.772, "latency": 62.075, "stderr": 0.972, "cost_per_test": 0.010812, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.0-flash-exp": { "accuracy": 60.374, "latency": 2.796, "stderr": 1.902, "cost_per_test": 0.000132, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 59.658, "latency": 3.794, "stderr": 1.91, "cost_per_test": 0.00027, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 59.26, "latency": 32.939, "stderr": 0.979, "cost_per_test": 0.001523, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 59.102, "latency": 14.526, "stderr": 0.964, "cost_per_test": 0.002585, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 58.586, "latency": 13.758, "stderr": 0.982, "cost_per_test": 0.08824, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 58.506, "latency": 2.716, "stderr": 0.967, "cost_per_test": 0.000752, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 58.426, "latency": 3.556, "stderr": 0.98, "cost_per_test": 0.001974, "temperature": 0.1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 57.75, "latency": 2.426, "stderr": 0.979, "cost_per_test": 0.000572, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 57.552, "latency": 28.206, "stderr": 0.985, "cost_per_test": 0.000286, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 57.512, "latency": 8.4, "stderr": 0.986, "cost_per_test": 0.0041, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 57.432, "latency": 5.984, "stderr": 1.907, "cost_per_test": 0.007141, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 56.756, "latency": 4.205, "stderr": 1.916, "cost_per_test": 0.002028, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 56.121, "latency": 23.52, "stderr": 0.988, "cost_per_test": 0.135806, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 55.803, "latency": 8.042, "stderr": 0.99, "cost_per_test": 0.000479, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 54.492, "latency": 6.059, "stderr": 1.93, "cost_per_test": 0.000822, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 53.617, "latency": 33.364, "stderr": 0.969, "cost_per_test": 0.002295, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 52.822, "latency": 3.858, "stderr": 0.995, "cost_per_test": 0.001003, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 52.106, "latency": 9.292, "stderr": 0.995, "cost_per_test": 0.003226, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 51.868, "latency": 25.948, "stderr": 0.993, "cost_per_test": 0.019966, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 49.88, "latency": 46.316, "stderr": 0.985, "cost_per_test": 0.052253, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 48.252, "latency": 443.266, "stderr": 0.973, "cost_per_test": 0.012184, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 45.35, "latency": 11.124, "stderr": 0.988, "cost_per_test": 0.024144, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 44.475, "latency": 65.022, "stderr": 0.963, "cost_per_test": 0.08575, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 42.766, "latency": 2.1, "stderr": 1.835, "cost_per_test": 0.00012, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 42.607, "latency": 47.45, "stderr": 0.952, "cost_per_test": 0.002438, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 42.09, "latency": 14.475, "stderr": 0.95, "cost_per_test": 0.004554, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo": { "accuracy": 38.792, "latency": 5.068, "stderr": 1.901, "cost_per_test": 0.000576, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 36.446, "latency": 7.138, "stderr": 0.756, "cost_per_test": 0.00269, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 34.26, "latency": 8.886, "stderr": 0.938, "cost_per_test": 0.001018, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 28.895, "latency": 68.43, "stderr": 0.838, "cost_per_test": 0.024032, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 25.318, "latency": 4.508, "stderr": 0.867, "cost_per_test": 0.003483, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo": { "accuracy": 23.291, "latency": 2.452, "stderr": 1.649, "cost_per_test": 9.2e-05, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-2-vision-1212": { "accuracy": 3.458, "latency": 0.0, "stderr": 0.719, "cost_per_test": 0.003772, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "semantic_extraction": { "anthropic/claude-fable-5-1": { "accuracy": 63.99, "latency": 12.176, "stderr": 1.353, "cost_per_test": 0.160752, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 62.719, "latency": 7.477, "stderr": 1.363, "cost_per_test": 0.072722, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 62.401, "latency": 22.444, "stderr": 1.366, "cost_per_test": 0.022352, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 62.162, "latency": 18.638, "stderr": 1.367, "cost_per_test": 0.031431, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 61.844, "latency": 12.409, "stderr": 1.37, "cost_per_test": 0.07514, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 61.685, "latency": 8.14, "stderr": 1.371, "cost_per_test": 0.005066, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 61.526, "latency": 3.902, "stderr": 1.372, "cost_per_test": 0.003276, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 61.367, "latency": 9.339, "stderr": 1.373, "cost_per_test": 0.049635, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 61.367, "latency": 17.152, "stderr": 1.373, "cost_per_test": 0.062828, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 61.129, "latency": 14.993, "stderr": 1.374, "cost_per_test": 0.166658, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 61.129, "latency": 25.492, "stderr": 1.374, "cost_per_test": 0.032418, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 60.811, "latency": 28.752, "stderr": 1.376, "cost_per_test": 0.011147, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 60.811, "latency": 117.863, "stderr": 1.376, "cost_per_test": 0.013533, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 60.652, "latency": 33.154, "stderr": 1.377, "cost_per_test": 0.075956, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.493, "latency": 13.769, "stderr": 1.378, "cost_per_test": 0.02657, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 60.493, "latency": 53.296, "stderr": 1.378, "cost_per_test": 0.007852, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 60.334, "latency": 5.612, "stderr": 1.379, "cost_per_test": 0.022016, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 60.254, "latency": 19.767, "stderr": 1.38, "cost_per_test": 0.004942, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 60.095, "latency": 6.99, "stderr": 1.381, "cost_per_test": 0.013193, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 60.095, "latency": 8.264, "stderr": 1.381, "cost_per_test": 0.025098, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 60.095, "latency": 11.794, "stderr": 1.381, "cost_per_test": 0.083407, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 60.095, "latency": 14.901, "stderr": 1.381, "cost_per_test": 0.041515, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 59.936, "latency": 9.991, "stderr": 1.382, "cost_per_test": 0.091307, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 59.777, "latency": 25.322, "stderr": 1.382, "cost_per_test": 0.001473, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 59.698, "latency": 7.092, "stderr": 1.383, "cost_per_test": 0.003816, "temperature": 0.1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 59.618, "latency": 6.156, "stderr": 2.707, "cost_per_test": 0.016315, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 59.459, "latency": 9.674, "stderr": 1.384, "cost_per_test": 0.062761, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 59.459, "latency": 14.499, "stderr": 1.384, "cost_per_test": 0.007033, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 59.141, "latency": 18.469, "stderr": 1.386, "cost_per_test": 0.135209, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 59.062, "latency": 4.413, "stderr": 1.386, "cost_per_test": 0.002448, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 58.983, "latency": 12.41, "stderr": 1.387, "cost_per_test": 0.00347, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 58.983, "latency": 303.455, "stderr": 1.387, "cost_per_test": 0.009623, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 58.824, "latency": 16.022, "stderr": 1.388, "cost_per_test": 0.109272, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 58.744, "latency": 10.793, "stderr": 1.388, "cost_per_test": 0.01655, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 58.665, "latency": 4.606, "stderr": 1.388, "cost_per_test": 0.002813, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 58.665, "latency": 10.824, "stderr": 1.388, "cost_per_test": 0.010586, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 58.665, "latency": 40.613, "stderr": 1.388, "cost_per_test": 0.007665, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 58.426, "latency": 7.245, "stderr": 1.39, "cost_per_test": 0.01573, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 58.426, "latency": 8.328, "stderr": 1.39, "cost_per_test": 0.016147, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 58.347, "latency": 34.036, "stderr": 1.39, "cost_per_test": 0.003226, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 58.188, "latency": 11.218, "stderr": 1.391, "cost_per_test": 0.082942, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 58.108, "latency": 55.639, "stderr": 1.391, "cost_per_test": 0.011447, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 57.949, "latency": 12.285, "stderr": 2.724, "cost_per_test": 0.022414, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 57.79, "latency": 18.935, "stderr": 1.392, "cost_per_test": 0.019246, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 57.79, "latency": 50.358, "stderr": 1.392, "cost_per_test": 0.009323, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 57.711, "latency": 6.149, "stderr": 1.393, "cost_per_test": 0.002525, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 57.711, "latency": 36.093, "stderr": 1.393, "cost_per_test": 0.030743, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 57.631, "latency": 4.041, "stderr": 1.393, "cost_per_test": 0.010562, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 57.552, "latency": 17.015, "stderr": 1.394, "cost_per_test": 0.008397, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 57.552, "latency": 41.625, "stderr": 1.394, "cost_per_test": 0.005853, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 57.472, "latency": 17.097, "stderr": 1.394, "cost_per_test": 0.025528, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 57.472, "latency": 27.66, "stderr": 1.394, "cost_per_test": 0.001423, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 57.234, "latency": 17.528, "stderr": 1.395, "cost_per_test": 0.013065, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 57.234, "latency": 34.203, "stderr": 1.395, "cost_per_test": 0.020691, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 57.154, "latency": 88.854, "stderr": 1.395, "cost_per_test": 0.020008, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 57.075, "latency": 28.359, "stderr": 1.396, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 56.995, "latency": 8.126, "stderr": 1.396, "cost_per_test": 0.082562, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 56.598, "latency": 25.266, "stderr": 1.397, "cost_per_test": 0.000286, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.0-flash-exp": { "accuracy": 56.518, "latency": 2.681, "stderr": 2.735, "cost_per_test": 0.00013, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 56.439, "latency": 2.448, "stderr": 1.398, "cost_per_test": 0.000336, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 56.359, "latency": 4.671, "stderr": 2.736, "cost_per_test": 0.000274, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 56.2, "latency": 33.613, "stderr": 1.399, "cost_per_test": 0.03448, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 56.041, "latency": 5.325, "stderr": 1.399, "cost_per_test": 0.001122, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 56.041, "latency": 32.629, "stderr": 1.399, "cost_per_test": 0.056618, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 55.723, "latency": 34.557, "stderr": 1.4, "cost_per_test": 0.023086, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 55.644, "latency": 5.23, "stderr": 1.401, "cost_per_test": 0.006366, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 54.928, "latency": 8.463, "stderr": 1.403, "cost_per_test": 0.001121, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 54.769, "latency": 2.644, "stderr": 1.403, "cost_per_test": 0.00133, "temperature": 0.1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 54.213, "latency": 8.719, "stderr": 1.405, "cost_per_test": 0.003196, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 54.134, "latency": 46.132, "stderr": 1.405, "cost_per_test": 0.020023, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 53.895, "latency": 43.542, "stderr": 1.405, "cost_per_test": 0.004693, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k3": { "accuracy": 53.816, "latency": 65.521, "stderr": 1.406, "cost_per_test": 0.027106, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 53.657, "latency": 7.079, "stderr": 1.406, "cost_per_test": 0.003826, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 52.464, "latency": 2.525, "stderr": 1.408, "cost_per_test": 0.000589, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 52.464, "latency": 9.297, "stderr": 1.408, "cost_per_test": 0.000499, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 51.272, "latency": 4.369, "stderr": 2.758, "cost_per_test": 0.015402, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 50.795, "latency": 3.889, "stderr": 1.41, "cost_per_test": 0.001009, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 50.795, "latency": 7.437, "stderr": 2.758, "cost_per_test": 0.007133, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 50.079, "latency": 4.421, "stderr": 2.759, "cost_per_test": 0.002137, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 50.079, "latency": 10.193, "stderr": 1.41, "cost_per_test": 0.002079, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 50.0, "latency": 6.119, "stderr": 2.759, "cost_per_test": 0.007347, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 49.841, "latency": 2.998, "stderr": 1.41, "cost_per_test": 0.000774, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 48.808, "latency": 6.187, "stderr": 2.758, "cost_per_test": 0.000916, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 48.251, "latency": 18.311, "stderr": 1.409, "cost_per_test": 0.017181, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 47.615, "latency": 104.197, "stderr": 1.408, "cost_per_test": 0.036726, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 47.536, "latency": 24.51, "stderr": 1.408, "cost_per_test": 0.013626, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 45.866, "latency": 61.393, "stderr": 1.405, "cost_per_test": 0.017204, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 42.369, "latency": 17.413, "stderr": 1.393, "cost_per_test": 0.001556, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.62, "latency": 9.149, "stderr": 1.385, "cost_per_test": 0.021303, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo": { "accuracy": 38.315, "latency": 5.819, "stderr": 2.683, "cost_per_test": 0.00063, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 29.65, "latency": 22.445, "stderr": 1.288, "cost_per_test": 0.001637, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 29.094, "latency": 8.631, "stderr": 1.281, "cost_per_test": 0.003893, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 28.06, "latency": 6.049, "stderr": 1.267, "cost_per_test": 0.00104, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 27.424, "latency": 2.197, "stderr": 2.463, "cost_per_test": 0.000129, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 25.437, "latency": 4.399, "stderr": 1.228, "cost_per_test": 0.003502, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo": { "accuracy": 25.199, "latency": 2.143, "stderr": 2.397, "cost_per_test": 8.6e-05, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 6.836, "latency": 7.507, "stderr": 0.712, "cost_per_test": 0.002722, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-2-vision-1212": { "accuracy": 2.385, "latency": 0.0, "stderr": 0.854, "cost_per_test": 0.00383, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "numerical_extraction": { "anthropic/claude-opus-5": { "accuracy": 81.399, "latency": 8.479, "stderr": 1.097, "cost_per_test": 0.074778, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 79.73, "latency": 29.835, "stderr": 1.133, "cost_per_test": 0.127846, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 79.412, "latency": 148.71, "stderr": 1.14, "cost_per_test": 0.052812, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 78.855, "latency": 92.365, "stderr": 1.151, "cost_per_test": 0.050856, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 78.696, "latency": 19.793, "stderr": 1.154, "cost_per_test": 0.093695, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 78.696, "latency": 47.997, "stderr": 1.154, "cost_per_test": 0.110772, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 78.14, "latency": 106.003, "stderr": 1.165, "cost_per_test": 0.017628, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 77.742, "latency": 5.183, "stderr": 2.297, "cost_per_test": 0.015277, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 77.583, "latency": 17.784, "stderr": 1.176, "cost_per_test": 0.182513, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 77.583, "latency": 28.249, "stderr": 1.176, "cost_per_test": 0.004647, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 77.504, "latency": 12.063, "stderr": 1.177, "cost_per_test": 0.019197, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 77.424, "latency": 11.076, "stderr": 1.179, "cost_per_test": 0.093589, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 77.266, "latency": 39.014, "stderr": 1.182, "cost_per_test": 0.009618, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 77.027, "latency": 6.323, "stderr": 1.186, "cost_per_test": 0.003233, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 76.948, "latency": 39.421, "stderr": 1.187, "cost_per_test": 0.079983, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 76.868, "latency": 4.079, "stderr": 2.328, "cost_per_test": 0.01475, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 76.789, "latency": 11.233, "stderr": 1.19, "cost_per_test": 0.023587, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 76.773, "latency": 79.525, "stderr": 1.058, "cost_per_test": 0.037789, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 76.709, "latency": 17.2, "stderr": 1.192, "cost_per_test": 0.183368, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 76.55, "latency": 105.239, "stderr": 1.195, "cost_per_test": 0.070212, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 76.391, "latency": 24.789, "stderr": 1.197, "cost_per_test": 0.024731, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 76.232, "latency": 4.837, "stderr": 1.2, "cost_per_test": 0.006059, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 76.232, "latency": 37.862, "stderr": 1.2, "cost_per_test": 0.221566, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.153, "latency": 39.334, "stderr": 1.201, "cost_per_test": 0.079986, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 75.994, "latency": 32.566, "stderr": 1.204, "cost_per_test": 0.050966, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 75.994, "latency": 56.849, "stderr": 1.204, "cost_per_test": 0.130934, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 75.835, "latency": 5.856, "stderr": 1.207, "cost_per_test": 0.00537, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.755, "latency": 14.885, "stderr": 2.366, "cost_per_test": 0.024887, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 75.755, "latency": 17.537, "stderr": 1.208, "cost_per_test": 0.009888, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 75.755, "latency": 55.395, "stderr": 1.208, "cost_per_test": 0.022666, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 75.676, "latency": 4.188, "stderr": 1.21, "cost_per_test": 0.01158, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 75.596, "latency": 33.26, "stderr": 1.211, "cost_per_test": 0.006139, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 75.437, "latency": 15.693, "stderr": 1.214, "cost_per_test": 0.005493, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 75.199, "latency": 67.998, "stderr": 1.218, "cost_per_test": 0.03919, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 75.119, "latency": 15.561, "stderr": 1.219, "cost_per_test": 0.079634, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 74.96, "latency": 45.711, "stderr": 1.221, "cost_per_test": 0.062771, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 74.96, "latency": 48.321, "stderr": 1.221, "cost_per_test": 0.002844, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 74.96, "latency": 103.935, "stderr": 1.221, "cost_per_test": 0.018388, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 74.801, "latency": 79.471, "stderr": 1.224, "cost_per_test": 0.009322, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 74.642, "latency": 10.727, "stderr": 1.227, "cost_per_test": 0.00346, "temperature": 0.1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 74.483, "latency": 30.7, "stderr": 1.229, "cost_per_test": 0.025379, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 74.483, "latency": 130.666, "stderr": 1.229, "cost_per_test": 0.037373, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 74.324, "latency": 7.021, "stderr": 1.232, "cost_per_test": 0.023396, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 73.847, "latency": 25.416, "stderr": 1.239, "cost_per_test": 0.011546, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 73.609, "latency": 55.043, "stderr": 1.243, "cost_per_test": 0.031564, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 72.337, "latency": 4.59, "stderr": 1.261, "cost_per_test": 0.002781, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 72.258, "latency": 60.441, "stderr": 1.262, "cost_per_test": 0.031612, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 71.781, "latency": 104.604, "stderr": 1.269, "cost_per_test": 0.078504, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 71.145, "latency": 7.274, "stderr": 2.501, "cost_per_test": 0.006779, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 70.986, "latency": 15.169, "stderr": 1.28, "cost_per_test": 0.012721, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 70.668, "latency": 180.885, "stderr": 1.284, "cost_per_test": 0.01836, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 70.588, "latency": 14.218, "stderr": 1.285, "cost_per_test": 0.005141, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 70.588, "latency": 21.324, "stderr": 1.285, "cost_per_test": 0.068848, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 69.952, "latency": 7.329, "stderr": 1.293, "cost_per_test": 0.000477, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 69.396, "latency": 106.608, "stderr": 1.299, "cost_per_test": 0.015356, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 69.157, "latency": 9.224, "stderr": 1.302, "cost_per_test": 0.001183, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 68.919, "latency": 28.199, "stderr": 1.305, "cost_per_test": 0.001186, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 68.283, "latency": 10.105, "stderr": 1.312, "cost_per_test": 0.01423, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 68.124, "latency": 18.86, "stderr": 1.314, "cost_per_test": 0.003092, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 67.17, "latency": 2.435, "stderr": 1.324, "cost_per_test": 0.00073, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 67.091, "latency": 42.056, "stderr": 1.325, "cost_per_test": 0.028418, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 66.852, "latency": 69.751, "stderr": 1.327, "cost_per_test": 0.071773, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 66.511, "latency": 8.517, "stderr": 1.317, "cost_per_test": 0.0158, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 66.057, "latency": 6.77, "stderr": 1.335, "cost_per_test": 0.002657, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 65.66, "latency": 55.497, "stderr": 1.339, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 65.183, "latency": 12.004, "stderr": 1.343, "cost_per_test": 0.093902, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 64.865, "latency": 5.85, "stderr": 2.634, "cost_per_test": 0.006935, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 64.865, "latency": 49.316, "stderr": 1.346, "cost_per_test": 0.003034, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.0-flash-exp": { "accuracy": 64.229, "latency": 2.911, "stderr": 2.645, "cost_per_test": 0.000135, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 63.434, "latency": 3.989, "stderr": 2.658, "cost_per_test": 0.001918, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 63.037, "latency": 2.328, "stderr": 1.361, "cost_per_test": 0.000554, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 62.957, "latency": 2.918, "stderr": 2.665, "cost_per_test": 0.000265, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 62.878, "latency": 83.537, "stderr": 1.362, "cost_per_test": 0.01396, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 62.798, "latency": 117.059, "stderr": 1.363, "cost_per_test": 0.016464, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 62.083, "latency": 4.467, "stderr": 1.368, "cost_per_test": 0.002617, "temperature": 0.1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 61.049, "latency": 38.218, "stderr": 1.375, "cost_per_test": 0.001623, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 60.175, "latency": 5.931, "stderr": 2.701, "cost_per_test": 0.000727, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 58.983, "latency": 16.297, "stderr": 1.387, "cost_per_test": 0.093539, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 58.506, "latency": 31.147, "stderr": 1.389, "cost_per_test": 0.000285, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 58.108, "latency": 2.002, "stderr": 2.722, "cost_per_test": 0.00011, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 57.313, "latency": 10.651, "stderr": 1.395, "cost_per_test": 0.005675, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 55.564, "latency": 72.456, "stderr": 1.401, "cost_per_test": 0.00324, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 55.485, "latency": 33.584, "stderr": 1.401, "cost_per_test": 0.022751, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 55.167, "latency": 13.637, "stderr": 1.402, "cost_per_test": 0.000622, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 55.087, "latency": 20.319, "stderr": 1.402, "cost_per_test": 0.005216, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 54.849, "latency": 3.828, "stderr": 1.403, "cost_per_test": 0.000997, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 53.418, "latency": 31.019, "stderr": 1.406, "cost_per_test": 0.162339, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 50.079, "latency": 13.1, "stderr": 1.41, "cost_per_test": 0.026984, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 50.0, "latency": 9.865, "stderr": 1.41, "cost_per_test": 0.003256, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 42.289, "latency": 75.535, "stderr": 1.393, "cost_per_test": 0.078978, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 40.461, "latency": 11.723, "stderr": 1.384, "cost_per_test": 0.000997, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo": { "accuracy": 39.269, "latency": 4.318, "stderr": 2.695, "cost_per_test": 0.000523, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4.3": { "accuracy": 37.52, "latency": 583.078, "stderr": 1.365, "cost_per_test": 0.014746, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 32.909, "latency": 97.415, "stderr": 1.325, "cost_per_test": 0.114883, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 25.199, "latency": 4.617, "stderr": 1.224, "cost_per_test": 0.003464, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo": { "accuracy": 21.383, "latency": 2.76, "stderr": 2.264, "cost_per_test": 9.7e-05, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 11.924, "latency": 75.468, "stderr": 0.914, "cost_per_test": 0.030861, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-2-vision-1212": { "accuracy": 4.531, "latency": 0.0, "stderr": 1.156, "cost_per_test": 0.003715, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } } } }