{ "metadata": { "benchmark": "Terminal-Bench 4.0", "slug": "terminal-bench-4", "description": "Frontier-difficulty terminal tasks across software, science, ML, operations, hardware, security, and media", "benchmark_id": "terminal_bench_4", "family": "terminal_bench", "version": "4.0", "updated": "2026-10-29", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "software": "Software", "science": "Science", "ml": "ML", "operations": "Operations", "hardware": "Hardware", "security": "Security", "media": "Media" }, "models": [ "alibaba/qwen3.8-max", "alibaba/qwen3.8-max-internal", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "baseten/nemotron-3.5-super-vl-120b-a12b", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "fireworks/ember-1", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "google/gemini-4-carbon", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "inception/mercury-3", "kimi/kimi-k3", "kimi/kimi-k3.1-pro", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M3", "minimax/MiniMax-M3.1-flash", "mistralai/mistral-large-4", "mistralai/mistral-large-4-oss", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "openai/gpt-6.1-terra", "reflection/Beam-1.0", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": true, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 66, "harness": "Mini-SWE-agent", "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 59 }, "tasks": { "overall": { "anthropic/claude-opus-5-5": { "accuracy": 65.152, "latency": 3810.852, "stderr": 0.0, "cost_per_test": 13.198632, "token_totals": { "input_tokens": 1850938631, "output_tokens": 14913437, "reasoning_tokens": 10690470, "cache_read_tokens": 1824613192, "cache_write_tokens": 26310456 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 112.2, "avg_query_seconds": 22.3, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 64.141, "latency": 4928.927, "stderr": 1.01, "cost_per_test": 16.513006, "token_totals": { "input_tokens": 3839535071, "output_tokens": 23509474, "reasoning_tokens": 17732859, "cache_read_tokens": 3801766853, "cache_write_tokens": 37747686 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 154.2, "avg_query_seconds": 19.2, "median_query_seconds": 9.4, "avg_ttft_seconds": 18.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 63.636, "latency": 2960.659, "stderr": 1.75, "cost_per_test": 4.56894, "token_totals": { "input_tokens": 1087338892, "output_tokens": 6110085, "reasoning_tokens": 3817315, "cache_read_tokens": 1074571435, "cache_write_tokens": 12131879 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 77.5, "avg_query_seconds": 29.7, "median_query_seconds": 18.0, "avg_ttft_seconds": 8.4, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-4-carbon": { "accuracy": 63.131, "latency": 2706.77, "stderr": 2.201, "cost_per_test": 7.452059, "token_totals": { "input_tokens": 812293708, "output_tokens": 10157385, "reasoning_tokens": 6013958, "cache_read_tokens": 779075436, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 59.596, "latency": 2142.082, "stderr": 4.403, "cost_per_test": 9.584259, "token_totals": { "input_tokens": 193308076, "output_tokens": 5366501, "reasoning_tokens": 3845988, "cache_read_tokens": 184898940, "cache_write_tokens": 8367663 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 29.8, "avg_query_seconds": 62.8, "median_query_seconds": 30.3, "avg_ttft_seconds": 45.1, "output_time_split": { "reasoning_pct": 5.7, "content_pct": 15.8, "tool_call_pct": 78.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 58.081, "latency": 3623.762, "stderr": 3.312, "cost_per_test": 17.175337, "token_totals": { "input_tokens": 1230443744, "output_tokens": 11653788, "reasoning_tokens": 7824002, "cache_read_tokens": 1208940872, "cache_write_tokens": 21489853 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 97.6, "avg_query_seconds": 25.5, "median_query_seconds": 10.4, "avg_ttft_seconds": 23.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 58.081, "latency": 4330.665, "stderr": 3.312, "cost_per_test": 10.528007, "token_totals": { "input_tokens": 2090144775, "output_tokens": 9938507, "reasoning_tokens": 6960300, "cache_read_tokens": 2069605468, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 139.8, "avg_query_seconds": 23.5, "median_query_seconds": 10.8, "avg_ttft_seconds": 21.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 57.576, "latency": 3214.793, "stderr": 2.314, "cost_per_test": 17.63994, "token_totals": { "input_tokens": 1088824555, "output_tokens": 10645668, "reasoning_tokens": 6605548, "cache_read_tokens": 895783049, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 73.5, "avg_query_seconds": 35.6, "median_query_seconds": 20.8, "avg_ttft_seconds": 12.6, "output_time_split": { "reasoning_pct": 65.7, "content_pct": 33.9, "tool_call_pct": 0.4 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 55.051, "latency": 2711.667, "stderr": 1.821, "cost_per_test": 1.72496, "token_totals": { "input_tokens": 182186678, "output_tokens": 6309120, "reasoning_tokens": 4640611, "cache_read_tokens": 172615332, "cache_write_tokens": 9543063 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 28.9, "avg_query_seconds": 81.9, "median_query_seconds": 37.2, "avg_ttft_seconds": 58.8, "output_time_split": { "reasoning_pct": 5.3, "content_pct": 14.8, "tool_call_pct": 79.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 53.535, "latency": 3995.725, "stderr": 1.336, "cost_per_test": 18.596913, "token_totals": { "input_tokens": 1552207398, "output_tokens": 12957487, "reasoning_tokens": 8759246, "cache_read_tokens": 1530055510, "cache_write_tokens": 22137527 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 108.8, "avg_query_seconds": 23.6, "median_query_seconds": 11.3, "avg_ttft_seconds": 21.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 49.495, "latency": 2032.802, "stderr": 1.821, "cost_per_test": 2.15663, "token_totals": { "input_tokens": 1102982070, "output_tokens": 7062468, "reasoning_tokens": 4559916, "cache_read_tokens": 1090297610, "cache_write_tokens": 12646620 }, "token_tasks_measured": 66, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 49.495, "latency": 3280.687, "stderr": 2.201, "cost_per_test": 14.657694, "token_totals": { "input_tokens": 2276758314, "output_tokens": 10781411, "reasoning_tokens": 7812084, "cache_read_tokens": 2198750960, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 146.6, "avg_query_seconds": 15.9, "median_query_seconds": 7.6, "avg_ttft_seconds": 14.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 44.444, "latency": 2096.128, "stderr": 3.535, "cost_per_test": 5.79322, "token_totals": { "input_tokens": 903418096, "output_tokens": 6801174, "reasoning_tokens": 4146060, "cache_read_tokens": 891860011, "cache_write_tokens": 11430701 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 74.3, "avg_query_seconds": 20.4, "median_query_seconds": 12.3, "avg_ttft_seconds": 10.0, "output_time_split": { "reasoning_pct": 17.2, "content_pct": 9.5, "tool_call_pct": 73.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 41.414, "latency": 4093.25, "stderr": 1.336, "cost_per_test": 30.339756, "token_totals": { "input_tokens": 1290713521, "output_tokens": 13036618, "reasoning_tokens": 9187742, "cache_read_tokens": 1266305087, "cache_write_tokens": 24396180 }, "token_tasks_measured": 66, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 38.889, "latency": 4952.947, "stderr": 1.821, "cost_per_test": 9.37436, "token_totals": { "input_tokens": 2068794376, "output_tokens": 12825626, "reasoning_tokens": 9581216, "cache_read_tokens": 2047400960, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 146.1, "avg_query_seconds": 20.0, "median_query_seconds": 9.3, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 98.2, "content_pct": 1.8, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 38.889, "latency": 8591.435, "stderr": 1.336, "cost_per_test": 12.146852, "token_totals": { "input_tokens": 2566775634, "output_tokens": 17868625, "reasoning_tokens": 13675033, "cache_read_tokens": 2536611883, "cache_write_tokens": null }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 141.0, "avg_query_seconds": 51.5, "median_query_seconds": 32.5, "avg_ttft_seconds": 27.5, "output_time_split": { "reasoning_pct": 79.8, "content_pct": 0.7, "tool_call_pct": 19.5 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 37.879, "latency": 2026.072, "stderr": 0.0, "cost_per_test": 7.983597, "token_totals": { "input_tokens": 687501203, "output_tokens": 4776552, "reasoning_tokens": 2335318, "cache_read_tokens": 676679299, "cache_write_tokens": 10768374 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 66.7, "avg_query_seconds": 16.6, "median_query_seconds": 9.6, "avg_ttft_seconds": 8.0, "output_time_split": { "reasoning_pct": 14.9, "content_pct": 34.1, "tool_call_pct": 51.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 37.374, "latency": 3681.073, "stderr": 2.672, "cost_per_test": 1.219907, "token_totals": { "input_tokens": 2341231330, "output_tokens": 12191289, "reasoning_tokens": 8946023, "cache_read_tokens": 2306387619, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 162.2, "avg_query_seconds": 12.6, "median_query_seconds": 5.3, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 78.3, "content_pct": 0.6, "tool_call_pct": 21.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 36.364, "latency": 3116.385, "stderr": 2.314, "cost_per_test": 3.117464, "token_totals": { "input_tokens": 2981637313, "output_tokens": 21606148, "reasoning_tokens": 16498863, "cache_read_tokens": 2952152829, "cache_write_tokens": 29466242 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 137.1, "avg_query_seconds": 11.3, "median_query_seconds": 4.7, "avg_ttft_seconds": 9.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 34.343, "latency": 8900.152, "stderr": 3.945, "cost_per_test": 10.633329, "token_totals": { "input_tokens": 2221841269, "output_tokens": 16292661, "reasoning_tokens": 12333902, "cache_read_tokens": 2194079317, "cache_write_tokens": null }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 132.6, "avg_query_seconds": 48.8, "median_query_seconds": 25.6, "avg_ttft_seconds": 9.0, "output_time_split": { "reasoning_pct": 78.4, "content_pct": 0.9, "tool_call_pct": 20.7 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 31.818, "latency": 2470.449, "stderr": 2.624, "cost_per_test": 0.539359, "token_totals": { "input_tokens": 2602850452, "output_tokens": 14064073, "reasoning_tokens": 10223328, "cache_read_tokens": 2592293631, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 162.5, "avg_query_seconds": 7.7, "median_query_seconds": 4.4, "avg_ttft_seconds": 1.7, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 31.818, "latency": 7066.968, "stderr": 2.624, "cost_per_test": 3.113224, "token_totals": { "input_tokens": 2678379838, "output_tokens": 15864425, "reasoning_tokens": null, "cache_read_tokens": 2648148395, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 157.7, "avg_query_seconds": 28.4, "median_query_seconds": 14.1, "avg_ttft_seconds": 6.8, "output_time_split": { "reasoning_pct": 80.9, "content_pct": 0.8, "tool_call_pct": 18.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 31.313, "latency": 9720.048, "stderr": 3.072, "cost_per_test": 0.456609, "token_totals": { "input_tokens": 1740494990, "output_tokens": 15117928, "reasoning_tokens": 12015515, "cache_read_tokens": 1715650773, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 108.9, "avg_query_seconds": 64.8, "median_query_seconds": 31.2, "avg_ttft_seconds": 16.7, "output_time_split": { "reasoning_pct": 87.3, "content_pct": 1.7, "tool_call_pct": 11.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 28.788, "latency": 3007.523, "stderr": 2.314, "cost_per_test": 18.011582, "token_totals": { "input_tokens": 1192820725, "output_tokens": 10497000, "reasoning_tokens": 6887226, "cache_read_tokens": 1145959595, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 95.2, "avg_query_seconds": 24.4, "median_query_seconds": 12.9, "avg_ttft_seconds": 4.1, "output_time_split": { "reasoning_pct": 94.7, "content_pct": 5.3, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 27.778, "latency": 6011.951, "stderr": 1.821, "cost_per_test": 5.180898, "token_totals": { "input_tokens": 2691212044, "output_tokens": 19160685, "reasoning_tokens": null, "cache_read_tokens": 2511343829, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 65, "avg_turns": 144.2, "avg_query_seconds": 29.5, "median_query_seconds": 9.6, "avg_ttft_seconds": 4.3, "output_time_split": { "reasoning_pct": 94.7, "content_pct": 5.2, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 26.263, "latency": 3872.678, "stderr": 5.126, "cost_per_test": 2.728364, "token_totals": { "input_tokens": 1641970464, "output_tokens": 10423768, "reasoning_tokens": null, "cache_read_tokens": 1573328896, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 25.253, "latency": 3030.551, "stderr": 1.01, "cost_per_test": 0.734722, "token_totals": { "input_tokens": 2497428556, "output_tokens": 16603505, "reasoning_tokens": 13158836, "cache_read_tokens": 2489516469, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 147.8, "avg_query_seconds": 9.9, "median_query_seconds": 5.0, "avg_ttft_seconds": 2.1, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 24.747, "latency": 3176.371, "stderr": 0.505, "cost_per_test": 6.654565, "token_totals": { "input_tokens": 2125885043, "output_tokens": 14326152, "reasoning_tokens": 9822430, "cache_read_tokens": 2071855568, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 24.242, "latency": 8712.24, "stderr": 1.515, "cost_per_test": 0.211804, "token_totals": { "input_tokens": 2175333414, "output_tokens": 16125095, "reasoning_tokens": 12753311, "cache_read_tokens": 2150748075, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 122.1, "avg_query_seconds": 49.3, "median_query_seconds": 23.2, "avg_ttft_seconds": 13.0, "output_time_split": { "reasoning_pct": 87.8, "content_pct": 1.2, "tool_call_pct": 11.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 23.232, "latency": 4283.164, "stderr": 1.336, "cost_per_test": 17.140717, "token_totals": { "input_tokens": 1116481668, "output_tokens": 16666742, "reasoning_tokens": 12858147, "cache_read_tokens": 1089283073, "cache_write_tokens": 27187396 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 84.8, "avg_query_seconds": 40.8, "median_query_seconds": 20.5, "avg_ttft_seconds": 34.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 22.727, "latency": 1888.568, "stderr": 2.314, "cost_per_test": 5.602118, "token_totals": { "input_tokens": 895160347, "output_tokens": 6238685, "reasoning_tokens": 3792707, "cache_read_tokens": 882387440, "cache_write_tokens": 12724594 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 83.1, "avg_query_seconds": 13.9, "median_query_seconds": 8.7, "avg_ttft_seconds": 6.8, "output_time_split": { "reasoning_pct": 31.2, "content_pct": 9.0, "tool_call_pct": 59.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 20.707, "latency": 6896.196, "stderr": 3.642, "cost_per_test": 19.484094, "token_totals": { "input_tokens": 3071210517, "output_tokens": 14252557, "reasoning_tokens": 10346067, "cache_read_tokens": 3015359154, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 156.9, "avg_query_seconds": 32.5, "median_query_seconds": 15.1, "avg_ttft_seconds": 4.5, "output_time_split": { "reasoning_pct": 74.3, "content_pct": 1.2, "tool_call_pct": 24.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 19.697, "latency": 2902.323, "stderr": 1.75, "cost_per_test": 0.499473, "token_totals": { "input_tokens": 2065521020, "output_tokens": 15407576, "reasoning_tokens": 12058588, "cache_read_tokens": 2058436005, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 19.697, "latency": 6427.575, "stderr": 2.314, "cost_per_test": 8.64749, "token_totals": { "input_tokens": 2400090539, "output_tokens": 12571537, "reasoning_tokens": 7741941, "cache_read_tokens": 2313842969, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 174.7, "avg_query_seconds": 6.1, "median_query_seconds": 4.5, "avg_ttft_seconds": 3.6, "output_time_split": { "reasoning_pct": 63.7, "content_pct": 34.5, "tool_call_pct": 1.7 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 18.687, "latency": 6368.24, "stderr": 3.535, "cost_per_test": 0.766472, "token_totals": { "input_tokens": 1973117830, "output_tokens": 15178104, "reasoning_tokens": 11818307, "cache_read_tokens": 1966243579, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 141.9, "avg_query_seconds": 12.0, "median_query_seconds": 8.3, "avg_ttft_seconds": 5.3, "output_time_split": { "reasoning_pct": 84.6, "content_pct": 0.7, "tool_call_pct": 14.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.6": { "accuracy": 17.172, "latency": 1765.468, "stderr": 1.336, "cost_per_test": 5.062449, "token_totals": { "input_tokens": 548390316, "output_tokens": 5919048, "reasoning_tokens": 3374782, "cache_read_tokens": 532115541, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 65.5, "avg_query_seconds": 19.9, "median_query_seconds": 10.0, "avg_ttft_seconds": 4.1, "output_time_split": { "reasoning_pct": 84.8, "content_pct": 15.2, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 17.172, "latency": 11353.52, "stderr": 0.505, "cost_per_test": 12.478176, "token_totals": { "input_tokens": 1806115861, "output_tokens": 12946019, "reasoning_tokens": 9401346, "cache_read_tokens": 1773695659, "cache_write_tokens": 31387110 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 124.1, "avg_query_seconds": 72.6, "median_query_seconds": 48.6, "avg_ttft_seconds": 27.6, "output_time_split": { "reasoning_pct": 77.6, "content_pct": 2.2, "tool_call_pct": 20.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 14.141, "latency": 5493.603, "stderr": 2.02, "cost_per_test": 3.311817, "token_totals": { "input_tokens": 3598466519, "output_tokens": 11740127, "reasoning_tokens": 6902591, "cache_read_tokens": 3587685589, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 232.3, "avg_query_seconds": 12.3, "median_query_seconds": 7.3, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 75.4, "content_pct": 0.7, "tool_call_pct": 23.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 13.636, "latency": 1965.568, "stderr": 1.515, "cost_per_test": 0.354726, "token_totals": { "input_tokens": 1004162895, "output_tokens": 10957435, "reasoning_tokens": 7328334, "cache_read_tokens": 988222853, "cache_write_tokens": 15670624 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 78.9, "avg_query_seconds": 16.1, "median_query_seconds": 8.0, "avg_ttft_seconds": 6.2, "output_time_split": { "reasoning_pct": 39.2, "content_pct": 8.8, "tool_call_pct": 51.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 12.121, "latency": 5026.717, "stderr": 3.03, "cost_per_test": 8.705677, "token_totals": { "input_tokens": 1529079806, "output_tokens": 10036181, "reasoning_tokens": 4455620, "cache_read_tokens": 1329123269, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 121.8, "avg_query_seconds": 11.2, "median_query_seconds": 8.2, "avg_ttft_seconds": 9.0, "output_time_split": { "reasoning_pct": 42.7, "content_pct": 55.8, "tool_call_pct": 1.5 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 11.616, "latency": 2155.227, "stderr": 1.01, "cost_per_test": 0.731646, "token_totals": { "input_tokens": 1246925576, "output_tokens": 6795331, "reasoning_tokens": 3907781, "cache_read_tokens": 1232687204, "cache_write_tokens": 14195105 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 109.6, "avg_query_seconds": 11.0, "median_query_seconds": 6.5, "avg_ttft_seconds": 6.0, "output_time_split": { "reasoning_pct": 24.6, "content_pct": 3.6, "tool_call_pct": 71.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 11.111, "latency": 5146.973, "stderr": 1.336, "cost_per_test": 3.452522, "token_totals": { "input_tokens": 3748960409, "output_tokens": 12276277, "reasoning_tokens": 7244257, "cache_read_tokens": 3737754965, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 227.2, "avg_query_seconds": 14.6, "median_query_seconds": 8.3, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 76.0, "content_pct": 0.8, "tool_call_pct": 23.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 10.606, "latency": 6297.51, "stderr": 1.515, "cost_per_test": 26.04996, "token_totals": { "input_tokens": 4437768112, "output_tokens": 20398255, "reasoning_tokens": 14862612, "cache_read_tokens": 4397229672, "cache_write_tokens": 40511971 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 200.5, "avg_query_seconds": 19.0, "median_query_seconds": 8.4, "avg_ttft_seconds": 17.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 10.606, "latency": 7112.43, "stderr": 1.75, "cost_per_test": 7.036467, "token_totals": { "input_tokens": 2197547446, "output_tokens": 13469736, "reasoning_tokens": 8997954, "cache_read_tokens": 2127067138, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 139.9, "avg_query_seconds": 19.8, "median_query_seconds": 12.1, "avg_ttft_seconds": 10.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 8.586, "latency": 2022.621, "stderr": 1.336, "cost_per_test": 5.51917, "token_totals": { "input_tokens": 630717218, "output_tokens": 5527326, "reasoning_tokens": 1268462, "cache_read_tokens": 611977984, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 74.8, "avg_query_seconds": 16.9, "median_query_seconds": 10.3, "avg_ttft_seconds": 4.3, "output_time_split": { "reasoning_pct": 32.5, "content_pct": 67.5, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 8.081, "latency": 7798.686, "stderr": 1.336, "cost_per_test": 3.07266, "token_totals": { "input_tokens": 2695499187, "output_tokens": 14630101, "reasoning_tokens": 10474030, "cache_read_tokens": 2628586667, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 154.5, "avg_query_seconds": 32.3, "median_query_seconds": 16.4, "avg_ttft_seconds": 6.6, "output_time_split": { "reasoning_pct": 78.0, "content_pct": 0.5, "tool_call_pct": 21.5 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-spark-open": { "accuracy": 6.566, "latency": 7466.403, "stderr": 0.505, "cost_per_test": 7.264062, "token_totals": { "input_tokens": 916170962, "output_tokens": 7264611, "reasoning_tokens": 4188707, "cache_read_tokens": 633327467, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 90.0, "avg_query_seconds": 71.5, "median_query_seconds": 26.6, "avg_ttft_seconds": 17.7, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 6.061, "latency": 4286.824, "stderr": 1.515, "cost_per_test": 5.981757, "token_totals": { "input_tokens": 1224247223, "output_tokens": 8938622, "reasoning_tokens": 5695714, "cache_read_tokens": 1127424034, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 107.6, "avg_query_seconds": 12.6, "median_query_seconds": 7.4, "avg_ttft_seconds": 8.1, "output_time_split": { "reasoning_pct": 69.2, "content_pct": 30.1, "tool_call_pct": 0.7 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 6.061, "latency": 4705.164, "stderr": 1.515, "cost_per_test": 4.135537, "token_totals": { "input_tokens": 1413112583, "output_tokens": 9461885, "reasoning_tokens": 4674885, "cache_read_tokens": 1394234793, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 110.3, "avg_query_seconds": 10.3, "median_query_seconds": 6.9, "avg_ttft_seconds": 7.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "reflection/Beam-1.0": { "accuracy": 4.04, "latency": 1154.552, "stderr": 1.336, "cost_per_test": null, "token_totals": { "input_tokens": 805547405, "output_tokens": 4342340, "reasoning_tokens": 2681354, "cache_read_tokens": 789244544, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 137.2, "avg_query_seconds": 1.6, "median_query_seconds": 0.8, "avg_ttft_seconds": 0.5, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 3.03, "latency": 1987.184, "stderr": 0.0, "cost_per_test": 0.421803, "token_totals": { "input_tokens": 1240859751, "output_tokens": 8811077, "reasoning_tokens": 8328363, "cache_read_tokens": 1103676197, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 136.2, "avg_query_seconds": 4.1, "median_query_seconds": 2.6, "avg_ttft_seconds": 3.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 2.525, "latency": 1124.176, "stderr": 0.505, "cost_per_test": 4.111957, "token_totals": { "input_tokens": 521456492, "output_tokens": 3925593, "reasoning_tokens": 2337576, "cache_read_tokens": 487469934, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 80.9, "avg_query_seconds": 9.3, "median_query_seconds": 6.3, "avg_ttft_seconds": 5.3, "output_time_split": { "reasoning_pct": 79.6, "content_pct": 19.6, "tool_call_pct": 0.8 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 2.525, "latency": 1770.424, "stderr": 1.01, "cost_per_test": 1.438224, "token_totals": { "input_tokens": 478181339, "output_tokens": 8640048, "reasoning_tokens": 6785249, "cache_read_tokens": 448286549, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 66.9, "avg_query_seconds": 16.5, "median_query_seconds": 8.3, "avg_ttft_seconds": 4.8, "output_time_split": { "reasoning_pct": 86.9, "content_pct": 1.4, "tool_call_pct": 11.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 1.01, "latency": 4582.924, "stderr": 1.01, "cost_per_test": 7.229497, "token_totals": { "input_tokens": 3308478889, "output_tokens": 12669342, "reasoning_tokens": 2749857, "cache_read_tokens": 3254206054, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 263.4, "avg_query_seconds": 8.4, "median_query_seconds": 5.3, "avg_ttft_seconds": 4.7, "output_time_split": { "reasoning_pct": 69.9, "content_pct": 4.3, "tool_call_pct": 25.9 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.505, "latency": 1378.754, "stderr": 0.505, "cost_per_test": 2.495998, "token_totals": { "input_tokens": 781467247, "output_tokens": 5735234, "reasoning_tokens": null, "cache_read_tokens": 771035008, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 140.2, "avg_query_seconds": 7.2, "median_query_seconds": 3.5, "avg_ttft_seconds": 2.5, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.505, "latency": 7622.037, "stderr": 0.505, "cost_per_test": 0.23303, "token_totals": { "input_tokens": 594781184, "output_tokens": 7209202, "reasoning_tokens": 4359700, "cache_read_tokens": 578632021, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 85.3, "avg_query_seconds": 45.8, "median_query_seconds": 17.3, "avg_ttft_seconds": 7.5, "output_time_split": { "reasoning_pct": 65.2, "content_pct": 1.2, "tool_call_pct": 33.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 705.115, "stderr": 0.0, "cost_per_test": 0.321679, "token_totals": { "input_tokens": 258225964, "output_tokens": 3667960, "reasoning_tokens": null, "cache_read_tokens": 252660395, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 91.0, "avg_query_seconds": 5.1, "median_query_seconds": 2.7, "avg_ttft_seconds": 2.2, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 0.0, "latency": 895.649, "stderr": 0.0, "cost_per_test": 0.862751, "token_totals": { "input_tokens": 970723485, "output_tokens": 10594448, "reasoning_tokens": 6900666, "cache_read_tokens": 890627349, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 913.643, "stderr": 0.0, "cost_per_test": 0.368017, "token_totals": { "input_tokens": 251527075, "output_tokens": 2283746, "reasoning_tokens": 2188216, "cache_read_tokens": 154050726, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 86.9, "avg_query_seconds": 3.8, "median_query_seconds": 2.8, "avg_ttft_seconds": 3.1, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "software": { "google/gemini-4-carbon": { "accuracy": 62.963, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 61.111, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 59.259, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 57.407, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 57.407, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 57.407, "latency": null, "stderr": 4.9, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 55.556, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 53.704, "latency": null, "stderr": 4.9, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 53.704, "latency": null, "stderr": 8.072, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 53.704, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 50.0, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 48.148, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 46.296, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 46.296, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 44.444, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 44.444, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 40.741, "latency": null, "stderr": 6.677, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 38.889, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 38.889, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 37.037, "latency": null, "stderr": 8.072, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 35.185, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 35.185, "latency": null, "stderr": 6.677, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 35.185, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3": { "accuracy": 35.185, "latency": null, "stderr": 4.9, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 35.185, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 33.333, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 29.63, "latency": null, "stderr": 4.9, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 29.63, "latency": null, "stderr": 4.9, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 29.63, "latency": null, "stderr": 4.9, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/ember-1": { "accuracy": 25.926, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.6": { "accuracy": 25.926, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 25.926, "latency": null, "stderr": 4.9, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 24.074, "latency": null, "stderr": 6.677, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 22.222, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 22.222, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 22.222, "latency": null, "stderr": 5.556, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 22.222, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 18.519, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 16.667, "latency": null, "stderr": 5.556, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 14.815, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 14.815, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 14.815, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 12.963, "latency": null, "stderr": 4.9, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 11.111, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 11.111, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 11.111, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "reflection/Beam-1.0": { "accuracy": 11.111, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 7.407, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 7.407, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 7.407, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 5.556, "latency": null, "stderr": 3.208, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 3.704, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 3.704, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 1.852, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 1.852, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 1.852, "latency": null, "stderr": 1.852, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "science": { "kimi/kimi-k3.1-pro": { "accuracy": 73.81, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 71.429, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-carbon": { "accuracy": 59.524, "latency": null, "stderr": 8.585, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 57.143, "latency": null, "stderr": 4.124, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 57.143, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 54.762, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 54.762, "latency": null, "stderr": 4.762, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 54.762, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 52.381, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 52.381, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 47.619, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 47.619, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 45.238, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 42.857, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 42.857, "latency": null, "stderr": 7.143, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 38.095, "latency": null, "stderr": 4.762, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 38.095, "latency": null, "stderr": 6.299, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 38.095, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 33.333, "latency": null, "stderr": 6.299, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "stepfun/step-5-preview": { "accuracy": 33.333, "latency": null, "stderr": 6.299, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 33.333, "latency": null, "stderr": 4.762, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 33.333, "latency": null, "stderr": 8.585, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 26.19, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 26.19, "latency": null, "stderr": 8.585, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 26.19, "latency": null, "stderr": 4.762, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 26.19, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 26.19, "latency": null, "stderr": 4.762, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 26.19, "latency": null, "stderr": 4.762, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 23.81, "latency": null, "stderr": 6.299, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 21.429, "latency": null, "stderr": 4.124, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 21.429, "latency": null, "stderr": 4.124, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 21.429, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 21.429, "latency": null, "stderr": 4.124, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 19.048, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 16.667, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 16.667, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 14.286, "latency": null, "stderr": 4.124, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 14.286, "latency": null, "stderr": 4.124, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 14.286, "latency": null, "stderr": 4.124, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 11.905, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 11.905, "latency": null, "stderr": 6.299, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 11.905, "latency": null, "stderr": 4.762, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 11.905, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 9.524, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 9.524, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 9.524, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 7.143, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 7.143, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 4.762, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 2.381, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 2.381, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 2.381, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 2.381, "latency": null, "stderr": 2.381, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null } }, "ml": { "anthropic/claude-opus-5-5": { "accuracy": 81.818, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 81.818, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 81.818, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-4-carbon": { "accuracy": 78.788, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 72.727, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 69.697, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 66.667, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 66.667, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 63.636, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 60.606, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 54.545, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 54.545, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 51.515, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 51.515, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 48.485, "latency": null, "stderr": 6.061, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-wonder": { "accuracy": 45.455, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 45.455, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 42.424, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 42.424, "latency": null, "stderr": 6.061, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 42.424, "latency": null, "stderr": 8.017, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 42.424, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 36.364, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 36.364, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 33.333, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 30.303, "latency": null, "stderr": 6.061, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "stepfun/step-5-preview": { "accuracy": 30.303, "latency": null, "stderr": 8.017, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 30.303, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 27.273, "latency": null, "stderr": 9.091, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 27.273, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 24.242, "latency": null, "stderr": 6.061, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 24.242, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 21.212, "latency": null, "stderr": 8.017, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 21.212, "latency": null, "stderr": 6.061, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 21.212, "latency": null, "stderr": 8.017, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 18.182, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 18.182, "latency": null, "stderr": 10.497, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 18.182, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 18.182, "latency": null, "stderr": 5.249, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 15.152, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.6": { "accuracy": 15.152, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 15.152, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 12.121, "latency": null, "stderr": 8.017, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 9.091, "latency": null, "stderr": 9.091, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 6.061, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-3": { "accuracy": 6.061, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 3.03, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 3.03, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 3.03, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 3.03, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "reflection/Beam-1.0": { "accuracy": 3.03, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "tencent/hy4-preview": { "accuracy": 3.03, "latency": null, "stderr": 3.03, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null } }, "operations": { "google/gemini-4-argon": { "accuracy": 55.556, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 55.556, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 51.852, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 48.148, "latency": null, "stderr": 13.354, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 48.148, "latency": null, "stderr": 9.799, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 44.444, "latency": null, "stderr": 11.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-carbon": { "accuracy": 44.444, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 40.741, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 40.741, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 40.741, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 40.741, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 40.741, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 29.63, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 25.926, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 25.926, "latency": null, "stderr": 9.799, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 25.926, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 25.926, "latency": null, "stderr": 9.799, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 25.926, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.6": { "accuracy": 22.222, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 22.222, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 22.222, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 18.519, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 18.519, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 18.519, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 18.519, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 18.519, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 14.815, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 14.815, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 14.815, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 14.815, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 14.815, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 11.111, "latency": null, "stderr": 11.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 11.111, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 11.111, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 11.111, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 11.111, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 11.111, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "stepfun/step-5-preview": { "accuracy": 11.111, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "tencent/hy4-preview": { "accuracy": 11.111, "latency": null, "stderr": 6.415, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 7.407, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 7.407, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 7.407, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 7.407, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 7.407, "latency": null, "stderr": 7.407, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 7.407, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 7.407, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 3.704, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 3.704, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 3.704, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-luna": { "accuracy": 3.704, "latency": null, "stderr": 3.704, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null } }, "hardware": { "anthropic/claude-fable-5-1": { "accuracy": 60.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 60.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-carbon": { "accuracy": 60.0, "latency": null, "stderr": 11.547, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 60.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 53.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 46.667, "latency": null, "stderr": 13.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 46.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 46.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 40.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 40.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 40.0, "latency": null, "stderr": 11.547, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 33.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 33.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 20.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 20.0, "latency": null, "stderr": 11.547, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 13.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 13.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 13.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 13.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "meta/muse-spark-open": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null } }, "security": { "anthropic/claude-opus-5": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 93.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 93.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 93.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 93.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 86.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 80.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 80.0, "latency": null, "stderr": 11.547, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 73.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-carbon": { "accuracy": 73.333, "latency": null, "stderr": 13.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 73.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 73.333, "latency": null, "stderr": 17.638, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 73.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "meta/muse-wonder": { "accuracy": 66.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 60.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 60.0, "latency": null, "stderr": 11.547, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 60.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 53.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 53.333, "latency": null, "stderr": 17.638, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 53.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 46.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 46.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 40.0, "latency": null, "stderr": 11.547, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 40.0, "latency": null, "stderr": 11.547, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 40.0, "latency": null, "stderr": 11.547, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 40.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 33.333, "latency": null, "stderr": 17.638, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 33.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 33.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 33.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 33.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.6": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 26.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 20.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 13.333, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "inception/mercury-3": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "meta/muse-spark-open": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 6.667, "latency": null, "stderr": 6.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null } }, "media": { "google/gemini-4-argon": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 66.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-carbon": { "accuracy": 66.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 66.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 58.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 58.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 58.333, "latency": null, "stderr": 16.667, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 58.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 50.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 50.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 50.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 50.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 41.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 41.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 41.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 33.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 33.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 25.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 25.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 25.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 25.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 25.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 25.0, "latency": null, "stderr": 14.434, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 25.0, "latency": null, "stderr": 14.434, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 25.0, "latency": null, "stderr": 14.434, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 16.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 16.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 16.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 16.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 16.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 16.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 16.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 8.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 8.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "meta/muse-spark-open": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null } } } }