{ "metadata": { "benchmark": "Terminal-Bench 2.0", "slug": "terminal-bench-2", "description": "State-of-the-art set of difficult terminal-based tasks", "benchmark_id": "terminal_bench_2", "family": "terminal_bench", "version": "2.0", "updated": "2026-06-04", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "easy": "Easy", "medium": "Medium", "hard": "Hard" }, "models": [ "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-max-preview", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "arcee-ai/trinity-large-thinking", "cohere/command-a-03-2025", "deepseek/deepseek-v4-pro", "fireworks/deepseek-v3p1", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/llama4-maverick-instruct-basic", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemma-4-31b-it", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "meta/muse_spark", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/mistral-large-2512", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2603", "openai/gpt-4.1-2025-04-14", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.3-codex", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "poolside/laguna-m.1", "poolside/laguna-xs.2", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 89, "runner": "external", "mode": "agentic", "archived": true, "partner": false, "total_models": 67 }, "tasks": { "overall": { "openai/gpt-5.5": { "accuracy": 73.202, "latency": 462.765, "stderr": 3.978, "cost_per_test": 0.760845, "token_totals": { "input_tokens": 20046148, "output_tokens": 1474787, "reasoning_tokens": 916212, "cache_read_tokens": 17057587, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 70.037, "latency": 472.445, "stderr": 1.35, "cost_per_test": 0.883864, "token_totals": { "input_tokens": 35316906, "output_tokens": 1733985, "reasoning_tokens": null, "cache_read_tokens": 32207503, "cache_write_tokens": 2930756 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 660.89, "stderr": 4.95, "cost_per_test": 0.975907, "token_totals": { "input_tokens": 38194626, "output_tokens": 1872733, "reasoning_tokens": null, "cache_read_tokens": 34512758, "cache_write_tokens": 3497323 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 67.416, "latency": 408.048, "stderr": 1.297, "cost_per_test": 1.192081, "token_totals": { "input_tokens": 217020103, "output_tokens": 2762736, "reasoning_tokens": 188175, "cache_read_tokens": 180962614, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 42.5, "avg_query_seconds": 4.5, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 67.416, "latency": 542.887, "stderr": 4.996, "cost_per_test": 0.516084, "token_totals": { "input_tokens": 19392790, "output_tokens": 2578482, "reasoning_tokens": 2089737, "cache_read_tokens": 13219922, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 64.045, "latency": 506.283, "stderr": 5.115, "cost_per_test": 0.345045, "token_totals": { "input_tokens": 18050124, "output_tokens": 1610657, "reasoning_tokens": 1098566, "cache_read_tokens": 14874880, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 59.551, "latency": 566.325, "stderr": 5.232, "cost_per_test": 0.070242, "token_totals": { "input_tokens": 36432732, "output_tokens": 1311034, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 59.551, "latency": 615.5, "stderr": 5.232, "cost_per_test": 0.518966, "token_totals": { "input_tokens": 48395130, "output_tokens": 1396087, "reasoning_tokens": null, "cache_read_tokens": 45244233, "cache_write_tokens": 2960900 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 59.176, "latency": 664.843, "stderr": 1.35, "cost_per_test": 0.719164, "token_totals": { "input_tokens": 33245136, "output_tokens": 2164716, "reasoning_tokens": 1631101, "cache_read_tokens": 15707819, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 14.1, "avg_query_seconds": 18.5, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 58.427, "latency": 627.493, "stderr": 5.254, "cost_per_test": 1.175583, "token_totals": { "input_tokens": 91703307, "output_tokens": 1390275, "reasoning_tokens": 0, "cache_read_tokens": 87379797, "cache_write_tokens": 3650078 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 58.427, "latency": 898.494, "stderr": 5.254, "cost_per_test": 1.140327, "token_totals": { "input_tokens": 31037223, "output_tokens": 2480166, "reasoning_tokens": null, "cache_read_tokens": 26753647, "cache_write_tokens": 3752177 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 58.427, "latency": 1027.517, "stderr": 5.254, "cost_per_test": 0.511959, "token_totals": { "input_tokens": 12725181, "output_tokens": 2287792, "reasoning_tokens": 1742395, "cache_read_tokens": 9140224, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 57.303, "latency": 795.458, "stderr": 5.273, "cost_per_test": 0.192786, "token_totals": { "input_tokens": 35008291, "output_tokens": 2465149, "reasoning_tokens": 0, "cache_read_tokens": 32861440, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 56.18, "latency": 828.342, "stderr": 5.289, "cost_per_test": 0.179637, "token_totals": { "input_tokens": 106542037, "output_tokens": 2145598, "reasoning_tokens": null, "cache_read_tokens": 104345088, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 55.056, "latency": 416.249, "stderr": 5.303, "cost_per_test": 0.455782, "token_totals": { "input_tokens": 23841106, "output_tokens": 2039873, "reasoning_tokens": 969892, "cache_read_tokens": 17553385, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 53.933, "latency": 661.913, "stderr": 5.314, "cost_per_test": 1.122133, "token_totals": { "input_tokens": 61554838, "output_tokens": 1845337, "reasoning_tokens": null, "cache_read_tokens": 57443298, "cache_write_tokens": 3565625 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 53.933, "latency": 881.675, "stderr": 5.314, "cost_per_test": 0.158286, "token_totals": { "input_tokens": 24198612, "output_tokens": 2440352, "reasoning_tokens": 1877168, "cache_read_tokens": 22400384, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 51.685, "latency": 456.071, "stderr": 5.327, "cost_per_test": 0.168702, "token_totals": { "input_tokens": 40847597, "output_tokens": 2706479, "reasoning_tokens": 764932, "cache_read_tokens": 30063957, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 51.685, "latency": 742.684, "stderr": 5.327, "cost_per_test": 0.469958, "token_totals": { "input_tokens": 8925515, "output_tokens": 2485285, "reasoning_tokens": 1876807, "cache_read_tokens": 5452288, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 51.685, "latency": 897.845, "stderr": 5.327, "cost_per_test": 0.320393, "token_totals": { "input_tokens": 14483532, "output_tokens": 1241842, "reasoning_tokens": 807225, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5-thinking": { "accuracy": 49.438, "latency": 875.602, "stderr": 5.33, "cost_per_test": 0.247234, "token_totals": { "input_tokens": 46819785, "output_tokens": 1665612, "reasoning_tokens": null, "cache_read_tokens": 37682368, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 47.191, "latency": 839.048, "stderr": 5.322, "cost_per_test": 0.073995, "token_totals": { "input_tokens": 93446002, "output_tokens": 1871910, "reasoning_tokens": null, "cache_read_tokens": 88994235, "cache_write_tokens": 4451767 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 46.067, "latency": 1019.457, "stderr": 5.314, "cost_per_test": 0.216209, "token_totals": { "input_tokens": 70621687, "output_tokens": 2241672, "reasoning_tokens": null, "cache_read_tokens": 59396792, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 26.2, "avg_query_seconds": 26.9, "median_query_seconds": 8.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 44.944, "latency": 747.185, "stderr": 4.492, "cost_per_test": 0.304702, "token_totals": { "input_tokens": 19782821, "output_tokens": 5068439, "reasoning_tokens": 4092186, "cache_read_tokens": 15594923, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 44.944, "latency": 840.666, "stderr": 5.303, "cost_per_test": 0.14597, "token_totals": { "input_tokens": 15053018, "output_tokens": 1821596, "reasoning_tokens": 1331434, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 44.944, "latency": 886.169, "stderr": 5.303, "cost_per_test": 0.407628, "token_totals": { "input_tokens": 8550221, "output_tokens": 3335632, "reasoning_tokens": 2926770, "cache_read_tokens": 6902400, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 44.944, "latency": 2007.631, "stderr": 5.303, "cost_per_test": 0.298145, "token_totals": { "input_tokens": 31188056, "output_tokens": 2172798, "reasoning_tokens": 1551825, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 43.446, "latency": 654.058, "stderr": 4.406, "cost_per_test": 0.373255, "token_totals": { "input_tokens": 64083766, "output_tokens": 2287763, "reasoning_tokens": 1365644, "cache_read_tokens": 56830387, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 41.573, "latency": 736.918, "stderr": 5.254, "cost_per_test": 0.751119, "token_totals": { "input_tokens": 59878600, "output_tokens": 2568420, "reasoning_tokens": null, "cache_read_tokens": 56989400, "cache_write_tokens": 2838435 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 41.573, "latency": 754.513, "stderr": 5.254, "cost_per_test": 0.085163, "token_totals": { "input_tokens": 85522946, "output_tokens": 1536657, "reasoning_tokens": null, "cache_read_tokens": 81679865, "cache_write_tokens": 3843081 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 41.573, "latency": 1877.151, "stderr": 5.254, "cost_per_test": 0.321994, "token_totals": { "input_tokens": 61061703, "output_tokens": 1763643, "reasoning_tokens": 821242, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 89, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.449, "latency": 161.408, "stderr": 5.232, "cost_per_test": 0.154109, "token_totals": { "input_tokens": 15258776, "output_tokens": 1198127, "reasoning_tokens": 869374, "cache_read_tokens": 13328128, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 40.449, "latency": 756.405, "stderr": 5.232, "cost_per_test": 0.139997, "token_totals": { "input_tokens": 68906339, "output_tokens": 1365271, "reasoning_tokens": null, "cache_read_tokens": 65959680, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 39.888, "latency": 845.812, "stderr": 4.454, "cost_per_test": 0.075077, "token_totals": { "input_tokens": 37132322, "output_tokens": 3849991, "reasoning_tokens": 2562050, "cache_read_tokens": 30872748, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 39.326, "latency": 855.298, "stderr": 5.207, "cost_per_test": null, "token_totals": { "input_tokens": 24362275, "output_tokens": 1829250, "reasoning_tokens": 1230171, "cache_read_tokens": 15461757, "cache_write_tokens": null }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 38.202, "latency": 655.044, "stderr": 5.18, "cost_per_test": 0.327722, "token_totals": { "input_tokens": 60901653, "output_tokens": 3877673, "reasoning_tokens": null, "cache_read_tokens": 57569275, "cache_write_tokens": 2758256 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.7": { "accuracy": 38.202, "latency": 777.512, "stderr": 5.18, "cost_per_test": 0.183983, "token_totals": { "input_tokens": 64638923, "output_tokens": 2190797, "reasoning_tokens": 1021645, "cache_read_tokens": 55568634, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 37.079, "latency": 782.07, "stderr": 5.149, "cost_per_test": 0.070371, "token_totals": { "input_tokens": 70175399, "output_tokens": 2308971, "reasoning_tokens": null, "cache_read_tokens": 66155073, "cache_write_tokens": 4020326 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 37.079, "latency": 986.102, "stderr": 5.149, "cost_per_test": 0.477064, "token_totals": { "input_tokens": 29311912, "output_tokens": 3629787, "reasoning_tokens": 2875648, "cache_read_tokens": 27092480, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 37.079, "latency": 986.553, "stderr": 5.149, "cost_per_test": 0.115876, "token_totals": { "input_tokens": 39185113, "output_tokens": 1348146, "reasoning_tokens": 0, "cache_read_tokens": 36818814, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 35.955, "latency": 1024.966, "stderr": 5.115, "cost_per_test": 0.166362, "token_totals": { "input_tokens": 32707341, "output_tokens": 2199105, "reasoning_tokens": null, "cache_read_tokens": 25729884, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 34.831, "latency": 804.541, "stderr": 5.079, "cost_per_test": 0.063298, "token_totals": { "input_tokens": 50556570, "output_tokens": 1557594, "reasoning_tokens": 0, "cache_read_tokens": 48650688, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 31.461, "latency": 0.0, "stderr": 4.95, "cost_per_test": null, "token_totals": { "input_tokens": 181126873, "output_tokens": 2003304, "reasoning_tokens": 441874, "cache_read_tokens": 87169715, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 45.2, "avg_query_seconds": 17.0, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 30.337, "latency": 796.526, "stderr": 4.901, "cost_per_test": 0.472003, "token_totals": { "input_tokens": 30291540, "output_tokens": 2914883, "reasoning_tokens": 2208351, "cache_read_tokens": 22226628, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 30.337, "latency": 1253.133, "stderr": 4.901, "cost_per_test": 4.710448, "token_totals": { "input_tokens": 259819365, "output_tokens": 3933438, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 29.213, "latency": 431.268, "stderr": 4.848, "cost_per_test": 0.03431, "token_totals": { "input_tokens": 17424104, "output_tokens": 2739625, "reasoning_tokens": 2050617, "cache_read_tokens": 12006889, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 28.09, "latency": 0.0, "stderr": 4.791, "cost_per_test": null, "token_totals": { "input_tokens": 517614743, "output_tokens": 3294845, "reasoning_tokens": 783659, "cache_read_tokens": 234488384, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 104.2, "avg_query_seconds": 8.4, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-4.6": { "accuracy": 28.09, "latency": 576.085, "stderr": 4.791, "cost_per_test": 0.14355, "token_totals": { "input_tokens": 79229320, "output_tokens": 917277, "reasoning_tokens": 44878, "cache_read_tokens": 75060428, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 28.09, "latency": 885.185, "stderr": 4.791, "cost_per_test": 0.552755, "token_totals": { "input_tokens": 10220735, "output_tokens": 2428513, "reasoning_tokens": 2069842, "cache_read_tokens": 7953197, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 26.966, "latency": 1028.621, "stderr": 4.731, "cost_per_test": 0.116731, "token_totals": { "input_tokens": 10972880, "output_tokens": 4722279, "reasoning_tokens": 3970304, "cache_read_tokens": 7994368, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 25.843, "latency": 702.2, "stderr": 4.667, "cost_per_test": 1.107135, "token_totals": { "input_tokens": 95271814, "output_tokens": 1087726, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 24.719, "latency": 479.157, "stderr": 4.599, "cost_per_test": 0.055526, "token_totals": { "input_tokens": 35415485, "output_tokens": 2123582, "reasoning_tokens": 1688469, "cache_read_tokens": 21353639, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 24.719, "latency": 667.879, "stderr": 4.599, "cost_per_test": 0.065496, "token_totals": { "input_tokens": 18699069, "output_tokens": 2452053, "reasoning_tokens": 2012988, "cache_read_tokens": 11216523, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 24.719, "latency": 716.499, "stderr": 4.599, "cost_per_test": 0.110414, "token_totals": { "input_tokens": 91151903, "output_tokens": 1779176, "reasoning_tokens": 720180, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 24.719, "latency": 912.424, "stderr": 4.599, "cost_per_test": 0.423036, "token_totals": { "input_tokens": 29273955, "output_tokens": 663276, "reasoning_tokens": 0, "cache_read_tokens": 10052352, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 22.472, "latency": 837.847, "stderr": 4.449, "cost_per_test": 0.501536, "token_totals": { "input_tokens": 77252430, "output_tokens": 818641, "reasoning_tokens": 0, "cache_read_tokens": 65768860, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 21.348, "latency": 417.621, "stderr": 4.368, "cost_per_test": 0.143966, "token_totals": { "input_tokens": 80482688, "output_tokens": 3015110, "reasoning_tokens": 1662269, "cache_read_tokens": 69887418, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 20.225, "latency": 831.981, "stderr": 4.282, "cost_per_test": 0.263469, "token_totals": { "input_tokens": 4265527, "output_tokens": 2569670, "reasoning_tokens": 2334053, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 19.101, "latency": 490.948, "stderr": 4.19, "cost_per_test": 0.131646, "token_totals": { "input_tokens": 74502471, "output_tokens": 901931, "reasoning_tokens": null, "cache_read_tokens": 47473369, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 17.978, "latency": 253.172, "stderr": 4.093, "cost_per_test": 0.032811, "token_totals": { "input_tokens": 33945065, "output_tokens": 503273, "reasoning_tokens": 0, "cache_read_tokens": 27469826, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 17.978, "latency": 1015.224, "stderr": 4.093, "cost_per_test": 0.325546, "token_totals": { "input_tokens": 236082030, "output_tokens": 4331418, "reasoning_tokens": 2598358, "cache_read_tokens": 178658880, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "mistralai/mistral-small-2603": { "accuracy": 16.854, "latency": 407.865, "stderr": 3.991, "cost_per_test": 0.105762, "token_totals": { "input_tokens": 55332312, "output_tokens": 1854888, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 14.607, "latency": 589.207, "stderr": 3.765, "cost_per_test": 1.078347, "token_totals": { "input_tokens": 125825982, "output_tokens": 967402, "reasoning_tokens": 0, "cache_read_tokens": 108945536, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 13.483, "latency": 898.203, "stderr": 3.641, "cost_per_test": 0.567192, "token_totals": { "input_tokens": 23153103, "output_tokens": 834776, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 8.989, "latency": 1180.302, "stderr": 3.049, "cost_per_test": 0.030482, "token_totals": { "input_tokens": 4238354, "output_tokens": 395836, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 2.247, "latency": 760.674, "stderr": 1.58, "cost_per_test": 2.436871, "token_totals": { "input_tokens": 82408379, "output_tokens": 1086058, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 2.247, "latency": 1193.58, "stderr": 1.58, "cost_per_test": 0.03402, "token_totals": { "input_tokens": 8777612, "output_tokens": 1246279, "reasoning_tokens": 0, "cache_read_tokens": 4923750, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "easy": { "grok/grok-4.20-0309-reasoning": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 85.0, "latency": null, "stderr": 15.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-m.1": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-4.6": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-4.7": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5-thinking": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.1": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 66.667, "latency": null, "stderr": 23.57, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2603": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 25.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null } }, "medium": { "openai/gpt-5.5": { "accuracy": 79.273, "latency": null, "stderr": 4.641, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 76.97, "latency": null, "stderr": 1.212, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 72.727, "latency": null, "stderr": 2.777, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 72.727, "latency": null, "stderr": 6.061, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 72.727, "latency": null, "stderr": 6.061, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 70.909, "latency": null, "stderr": 6.181, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 69.091, "latency": null, "stderr": 6.289, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 69.091, "latency": null, "stderr": 6.289, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 69.091, "latency": null, "stderr": 6.289, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 69.091, "latency": null, "stderr": 6.289, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 69.091, "latency": null, "stderr": 6.289, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 67.879, "latency": null, "stderr": 2.424, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 67.273, "latency": null, "stderr": 6.385, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 67.273, "latency": null, "stderr": 6.385, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 67.273, "latency": null, "stderr": 6.385, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 63.636, "latency": null, "stderr": 6.546, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 61.818, "latency": null, "stderr": 6.611, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 61.818, "latency": null, "stderr": 6.611, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.1": { "accuracy": 61.818, "latency": null, "stderr": 6.611, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 60.0, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 56.364, "latency": null, "stderr": 6.749, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 56.364, "latency": null, "stderr": 6.749, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 53.939, "latency": null, "stderr": 5.572, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 52.727, "latency": null, "stderr": 6.794, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 52.727, "latency": null, "stderr": 6.794, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 52.727, "latency": null, "stderr": 6.794, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 52.727, "latency": null, "stderr": 6.794, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 52.727, "latency": null, "stderr": 6.794, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 52.121, "latency": null, "stderr": 5.722, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 50.909, "latency": null, "stderr": 6.803, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 49.091, "latency": null, "stderr": 6.803, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 49.091, "latency": null, "stderr": 6.803, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 49.091, "latency": null, "stderr": 6.803, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 49.091, "latency": null, "stderr": 6.803, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 47.273, "latency": null, "stderr": 5.652, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 47.273, "latency": null, "stderr": 6.794, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 47.273, "latency": null, "stderr": 6.794, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 47.273, "latency": null, "stderr": 6.794, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 45.455, "latency": null, "stderr": 6.776, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 45.455, "latency": null, "stderr": 6.776, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 43.636, "latency": null, "stderr": 6.749, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 43.636, "latency": null, "stderr": 6.749, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 41.818, "latency": null, "stderr": 6.712, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 38.182, "latency": null, "stderr": 6.611, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 38.182, "latency": null, "stderr": 6.611, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 36.364, "latency": null, "stderr": 6.546, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 34.545, "latency": null, "stderr": 6.471, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-4.6": { "accuracy": 34.545, "latency": null, "stderr": 6.471, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 34.545, "latency": null, "stderr": 6.471, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 32.727, "latency": null, "stderr": 6.385, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 32.727, "latency": null, "stderr": 6.385, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 30.909, "latency": null, "stderr": 6.289, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 29.091, "latency": null, "stderr": 6.181, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 29.091, "latency": null, "stderr": 6.181, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 29.091, "latency": null, "stderr": 6.181, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 27.273, "latency": null, "stderr": 6.061, "cost_per_test": null, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 25.455, "latency": null, "stderr": 5.928, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 21.818, "latency": null, "stderr": 5.62, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2603": { "accuracy": 21.818, "latency": null, "stderr": 5.62, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 21.818, "latency": null, "stderr": 5.62, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 20.0, "latency": null, "stderr": 5.443, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 20.0, "latency": null, "stderr": 5.443, "cost_per_test": null, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 18.182, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 18.182, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 10.909, "latency": null, "stderr": 4.242, "cost_per_test": null, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 1.818, "latency": null, "stderr": 1.818, "cost_per_test": null, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 1.818, "latency": null, "stderr": 1.818, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "hard": { "openai/gpt-5.5": { "accuracy": 60.5, "latency": null, "stderr": 7.567, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 56.667, "latency": null, "stderr": 9.202, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 54.444, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 53.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 53.333, "latency": null, "stderr": 9.264, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 50.0, "latency": null, "stderr": 9.285, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 50.0, "latency": null, "stderr": 9.285, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 43.333, "latency": null, "stderr": 9.202, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark": { "accuracy": 40.0, "latency": null, "stderr": 9.097, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 40.0, "latency": null, "stderr": 9.097, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 40.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 36.667, "latency": null, "stderr": 8.949, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 36.667, "latency": null, "stderr": 8.949, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.1": { "accuracy": 36.667, "latency": null, "stderr": 8.949, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 33.333, "latency": null, "stderr": 8.754, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 33.333, "latency": null, "stderr": 8.754, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 33.333, "latency": null, "stderr": 8.754, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 33.333, "latency": null, "stderr": 8.754, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 30.0, "latency": null, "stderr": 8.51, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 26.667, "latency": null, "stderr": 8.212, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 26.667, "latency": null, "stderr": 8.212, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 26.667, "latency": null, "stderr": 8.212, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 24.444, "latency": null, "stderr": 5.966, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 24.444, "latency": null, "stderr": 6.577, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 23.333, "latency": null, "stderr": 7.854, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.7": { "accuracy": 23.333, "latency": null, "stderr": 7.854, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 23.333, "latency": null, "stderr": 7.854, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5-thinking": { "accuracy": 23.333, "latency": null, "stderr": 7.854, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 21.667, "latency": null, "stderr": 6.35, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 20.0, "latency": null, "stderr": 7.428, "cost_per_test": null, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 20.0, "latency": null, "stderr": 7.428, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 20.0, "latency": null, "stderr": 7.428, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.0, "latency": null, "stderr": 7.428, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 20.0, "latency": null, "stderr": 7.428, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 16.667, "latency": null, "stderr": 6.92, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 16.667, "latency": null, "stderr": 6.92, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 16.667, "latency": null, "stderr": 6.92, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 16.667, "latency": null, "stderr": 6.92, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 16.667, "latency": null, "stderr": 6.92, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 16.667, "latency": null, "stderr": 6.92, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-m.1": { "accuracy": 13.333, "latency": null, "stderr": 6.312, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 13.333, "latency": null, "stderr": 6.312, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 13.333, "latency": null, "stderr": 6.312, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 13.333, "latency": null, "stderr": 6.312, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 13.333, "latency": null, "stderr": 6.312, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 13.333, "latency": null, "stderr": 6.312, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 13.333, "latency": null, "stderr": 6.312, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 13.333, "latency": null, "stderr": 6.312, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 10.0, "latency": null, "stderr": 5.571, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-4.6": { "accuracy": 10.0, "latency": null, "stderr": 5.571, "cost_per_test": null, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 10.0, "latency": null, "stderr": 5.571, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 10.0, "latency": null, "stderr": 5.571, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 10.0, "latency": null, "stderr": 5.571, "cost_per_test": null, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 6.667, "latency": null, "stderr": 4.632, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 6.667, "latency": null, "stderr": 4.632, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 6.667, "latency": null, "stderr": 4.632, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 6.667, "latency": null, "stderr": 4.632, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 6.667, "latency": null, "stderr": 4.632, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2603": { "accuracy": 3.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 3.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 3.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 3.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 3.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 3.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }