{ "metadata": { "benchmark": "Immigration", "slug": "immigration", "description": "Evaluating agents on SME-authored US immigration law questions graded against per-question rubrics", "benchmark_id": "immigration", "family": "immigration", "version": "1", "updated": "2026-10-28", "dataset_type": "private", "industry": "legal", "tasks": { "overall": "Overall", "all_pass": "All-Pass", "intake": "Intake Checks", "content": "Content Checks" }, "models": [ "alibaba/qwen3.8-max", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.6", "grok/grok-4.7", "kimi/kimi-k3", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M3", "mistralai/mistral-large-4", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6.1-sol", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 25 }, "tasks": { "overall": { "anthropic/claude-fable-5-1": { "accuracy": 75.444, "latency": 435.009, "stderr": 1.4, "cost_per_test": 2.106027, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 72.14, "latency": 800.063, "stderr": 1.448, "cost_per_test": 2.164083, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 71.221, "latency": 551.594, "stderr": 1.464, "cost_per_test": 0.388431, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 70.139, "latency": 264.993, "stderr": 1.571, "cost_per_test": 0.116743, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-wonder": { "accuracy": 69.616, "latency": 172.922, "stderr": 1.437, "cost_per_test": 0.302474, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 68.962, "latency": 662.52, "stderr": 1.542, "cost_per_test": 1.159238, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 68.256, "latency": 64.84, "stderr": 1.515, "cost_per_test": 0.17986, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 68.075, "latency": 308.893, "stderr": 1.454, "cost_per_test": 0.5065, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 68.049, "latency": 367.406, "stderr": 1.603, "cost_per_test": 0.237598, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 67.807, "latency": 470.102, "stderr": 1.535, "cost_per_test": 0.232235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 67.691, "latency": 501.41, "stderr": 1.548, "cost_per_test": 1.373884, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 67.501, "latency": 325.1, "stderr": 1.553, "cost_per_test": 0.016814, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 66.731, "latency": 343.733, "stderr": 1.622, "cost_per_test": 0.124343, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 66.542, "latency": 39.851, "stderr": 1.589, "cost_per_test": 0.039371, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 65.982, "latency": 219.773, "stderr": 1.535, "cost_per_test": 0.100189, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 65.638, "latency": 225.22, "stderr": 1.638, "cost_per_test": 0.053824, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 64.997, "latency": 79.863, "stderr": 1.564, "cost_per_test": 0.045526, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 64.929, "latency": 97.631, "stderr": 1.565, "cost_per_test": 0.059656, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 63.708, "latency": 554.005, "stderr": 1.659, "cost_per_test": 0.146037, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 63.405, "latency": 56.532, "stderr": 1.613, "cost_per_test": 0.16645, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 62.529, "latency": 150.535, "stderr": 1.572, "cost_per_test": 0.010756, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 61.791, "latency": 399.438, "stderr": 1.637, "cost_per_test": 0.089748, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 61.66, "latency": 133.4, "stderr": 1.681, "cost_per_test": 0.051477, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 57.526, "latency": 99.375, "stderr": 1.616, "cost_per_test": 0.03662, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 55.661, "latency": 25.176, "stderr": 1.703, "cost_per_test": 0.024107, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "all_pass": { "anthropic/claude-fable-5-1": { "accuracy": 26.336, "latency": 435.009, "stderr": 2.726, "cost_per_test": 2.106027, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 25.573, "latency": 551.594, "stderr": 2.7, "cost_per_test": 0.388431, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 25.191, "latency": 264.993, "stderr": 2.687, "cost_per_test": 0.116743, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 24.427, "latency": 800.063, "stderr": 2.66, "cost_per_test": 2.164083, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 22.519, "latency": 367.406, "stderr": 2.586, "cost_per_test": 0.237598, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 22.519, "latency": 470.102, "stderr": 2.586, "cost_per_test": 0.232235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 22.137, "latency": 501.41, "stderr": 2.57, "cost_per_test": 1.373884, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 22.137, "latency": 662.52, "stderr": 2.57, "cost_per_test": 1.159238, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 21.756, "latency": 172.922, "stderr": 2.554, "cost_per_test": 0.302474, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 21.374, "latency": 39.851, "stderr": 2.538, "cost_per_test": 0.039371, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 21.374, "latency": 64.84, "stderr": 2.538, "cost_per_test": 0.17986, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 20.992, "latency": 343.733, "stderr": 2.521, "cost_per_test": 0.124343, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.611, "latency": 308.893, "stderr": 2.504, "cost_per_test": 0.5065, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.611, "latency": 325.1, "stderr": 2.504, "cost_per_test": 0.016814, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 19.466, "latency": 225.22, "stderr": 2.451, "cost_per_test": 0.053824, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 18.702, "latency": 554.005, "stderr": 2.414, "cost_per_test": 0.146037, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 18.321, "latency": 79.863, "stderr": 2.394, "cost_per_test": 0.045526, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 17.557, "latency": 219.773, "stderr": 2.355, "cost_per_test": 0.100189, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 17.176, "latency": 97.631, "stderr": 2.335, "cost_per_test": 0.059656, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 17.176, "latency": 133.4, "stderr": 2.335, "cost_per_test": 0.051477, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 16.412, "latency": 56.532, "stderr": 2.293, "cost_per_test": 0.16645, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 16.412, "latency": 150.535, "stderr": 2.293, "cost_per_test": 0.010756, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 16.031, "latency": 399.438, "stderr": 2.271, "cost_per_test": 0.089748, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 14.122, "latency": 25.176, "stderr": 2.156, "cost_per_test": 0.024107, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 12.595, "latency": 99.375, "stderr": 2.054, "cost_per_test": 0.03662, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null } }, "intake": { "anthropic/claude-fable-5-1": { "accuracy": 58.904, "latency": 435.009, "stderr": 0.0, "cost_per_test": 2.106027, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 49.664, "latency": 800.063, "stderr": 0.0, "cost_per_test": 2.164083, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 48.322, "latency": 264.993, "stderr": 0.0, "cost_per_test": 0.116743, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-wonder-full": { "accuracy": 48.322, "latency": 551.594, "stderr": 0.0, "cost_per_test": 0.388431, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 47.651, "latency": 172.922, "stderr": 0.0, "cost_per_test": 0.302474, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 47.651, "latency": 343.733, "stderr": 0.0, "cost_per_test": 0.124343, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 47.651, "latency": 399.438, "stderr": 0.0, "cost_per_test": 0.089748, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 47.651, "latency": 662.52, "stderr": 0.0, "cost_per_test": 1.159238, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 45.638, "latency": 325.1, "stderr": 0.0, "cost_per_test": 0.016814, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-4-argon": { "accuracy": 44.966, "latency": 308.893, "stderr": 0.0, "cost_per_test": 0.5065, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 43.537, "latency": 133.4, "stderr": 0.0, "cost_per_test": 0.051477, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 42.953, "latency": 64.84, "stderr": 0.0, "cost_per_test": 0.17986, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 42.953, "latency": 79.863, "stderr": 0.0, "cost_per_test": 0.045526, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 42.282, "latency": 501.41, "stderr": 0.0, "cost_per_test": 1.373884, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 41.611, "latency": 367.406, "stderr": 0.0, "cost_per_test": 0.237598, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 40.94, "latency": 56.532, "stderr": 0.0, "cost_per_test": 0.16645, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 40.268, "latency": 219.773, "stderr": 0.0, "cost_per_test": 0.100189, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 39.597, "latency": 225.22, "stderr": 0.0, "cost_per_test": 0.053824, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 39.597, "latency": 470.102, "stderr": 0.0, "cost_per_test": 0.232235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 39.597, "latency": 554.005, "stderr": 0.0, "cost_per_test": 0.146037, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 38.926, "latency": 39.851, "stderr": 0.0, "cost_per_test": 0.039371, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 38.926, "latency": 97.631, "stderr": 0.0, "cost_per_test": 0.059656, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-luna": { "accuracy": 38.926, "latency": 150.535, "stderr": 0.0, "cost_per_test": 0.010756, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 36.913, "latency": 99.375, "stderr": 0.0, "cost_per_test": 0.03662, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 31.544, "latency": 25.176, "stderr": 0.0, "cost_per_test": 0.024107, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "content": { "anthropic/claude-fable-5-1": { "accuracy": 73.155, "latency": 435.009, "stderr": 0.0, "cost_per_test": 2.106027, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.237, "latency": 800.063, "stderr": 0.0, "cost_per_test": 2.164083, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 68.731, "latency": 551.594, "stderr": 0.0, "cost_per_test": 0.388431, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.871, "latency": 264.993, "stderr": 0.0, "cost_per_test": 0.116743, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-wonder": { "accuracy": 67.355, "latency": 172.922, "stderr": 0.0, "cost_per_test": 0.302474, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 66.667, "latency": 367.406, "stderr": 0.0, "cost_per_test": 0.237598, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 66.151, "latency": 662.52, "stderr": 0.0, "cost_per_test": 1.159238, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 65.72, "latency": 325.1, "stderr": 0.0, "cost_per_test": 0.016814, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 65.634, "latency": 64.84, "stderr": 0.0, "cost_per_test": 0.17986, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 65.591, "latency": 308.893, "stderr": 0.0, "cost_per_test": 0.5065, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 65.118, "latency": 470.102, "stderr": 0.0, "cost_per_test": 0.232235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 64.86, "latency": 501.41, "stderr": 0.0, "cost_per_test": 1.373884, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 64.473, "latency": 343.733, "stderr": 0.0, "cost_per_test": 0.124343, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 64.344, "latency": 39.851, "stderr": 0.0, "cost_per_test": 0.039371, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 64.215, "latency": 225.22, "stderr": 0.0, "cost_per_test": 0.053824, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.6": { "accuracy": 63.312, "latency": 219.773, "stderr": 0.0, "cost_per_test": 0.100189, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 62.323, "latency": 97.631, "stderr": 0.0, "cost_per_test": 0.059656, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 62.194, "latency": 79.863, "stderr": 0.0, "cost_per_test": 0.045526, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 61.247, "latency": 56.532, "stderr": 0.0, "cost_per_test": 0.16645, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 60.774, "latency": 150.535, "stderr": 0.0, "cost_per_test": 0.010756, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 60.172, "latency": 554.005, "stderr": 0.0, "cost_per_test": 0.146037, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 59.146, "latency": 133.4, "stderr": 0.0, "cost_per_test": 0.051477, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 57.591, "latency": 399.438, "stderr": 0.0, "cost_per_test": 0.089748, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 56.215, "latency": 99.375, "stderr": 0.0, "cost_per_test": 0.03662, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 51.742, "latency": 25.176, "stderr": 0.0, "cost_per_test": 0.024107, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } } } }