{ "metadata": { "benchmark": "Financial Coaching Bench", "slug": "financial-coaching-bench", "description": "Can AI agents coach a simulated client through a real financial health problem?", "benchmark_id": "financial_health", "family": "financial_health", "version": "1", "updated": "2026-11-13", "dataset_type": "private", "industry": "social mobility", "tasks": { "overall": "Overall", "length_controlled": "Length-Controlled", "elicitation": "Elicitation" }, "models": [ "alibaba/qwen3.8-max", "anthropic/claude-fable-5-1", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4.1-flash", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.7", "kimi/kimi-k3", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M3", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 28 }, "tasks": { "overall": { "anthropic/claude-opus-5-5": { "accuracy": 76.162, "latency": 652.03, "stderr": 1.519, "cost_per_test": 1.633943, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 75.738, "latency": 191.188, "stderr": 1.549, "cost_per_test": 0.05932, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 75.711, "latency": 348.037, "stderr": 1.539, "cost_per_test": 1.14488, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 75.576, "latency": 353.159, "stderr": 1.502, "cost_per_test": 0.127988, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 75.37, "latency": 379.243, "stderr": 1.44, "cost_per_test": 0.013134, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder-full": { "accuracy": 74.963, "latency": 1026.858, "stderr": 1.572, "cost_per_test": 0.444098, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 74.943, "latency": 525.253, "stderr": 1.55, "cost_per_test": 0.80344, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 74.892, "latency": 369.608, "stderr": 1.515, "cost_per_test": 0.469621, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 74.526, "latency": 508.63, "stderr": 1.494, "cost_per_test": 0.190917, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 74.07, "latency": 311.74, "stderr": 1.548, "cost_per_test": 0.237214, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 74.055, "latency": 337.699, "stderr": 1.457, "cost_per_test": 0.094398, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-wonder": { "accuracy": 73.783, "latency": 250.64, "stderr": 1.555, "cost_per_test": 0.320778, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 73.132, "latency": 288.391, "stderr": 1.471, "cost_per_test": 0.15542, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 72.833, "latency": 367.969, "stderr": 1.569, "cost_per_test": 0.015189, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 70.988, "latency": 192.151, "stderr": 1.535, "cost_per_test": 0.070122, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 70.977, "latency": 322.4, "stderr": 1.54, "cost_per_test": 0.307623, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 70.287, "latency": 177.751, "stderr": 1.675, "cost_per_test": 0.085966, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 69.286, "latency": 220.97, "stderr": 1.656, "cost_per_test": 0.023081, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 69.049, "latency": 169.099, "stderr": 1.584, "cost_per_test": 0.026765, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 68.548, "latency": 670.02, "stderr": 1.598, "cost_per_test": 0.295262, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 67.534, "latency": 753.026, "stderr": 1.688, "cost_per_test": 1.72152, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 66.729, "latency": 443.204, "stderr": 1.502, "cost_per_test": 0.122276, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 66.272, "latency": 115.204, "stderr": 1.577, "cost_per_test": 0.09976, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 65.763, "latency": 136.024, "stderr": 1.565, "cost_per_test": 0.100547, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 65.241, "latency": 349.074, "stderr": 1.682, "cost_per_test": 0.020924, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 62.958, "latency": 140.018, "stderr": 1.657, "cost_per_test": 0.128701, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 62.614, "latency": 411.312, "stderr": 1.717, "cost_per_test": 0.233012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 57.465, "latency": 94.051, "stderr": 1.699, "cost_per_test": 0.025801, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "length_controlled": { "openai/gpt-5.6-sol": { "accuracy": 76.046, "latency": 369.608, "stderr": 1.515, "cost_per_test": 0.469621, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.899, "latency": 652.03, "stderr": 1.519, "cost_per_test": 1.633943, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 75.566, "latency": 348.037, "stderr": 1.539, "cost_per_test": 1.14488, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.165, "latency": 525.253, "stderr": 1.55, "cost_per_test": 0.80344, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 74.768, "latency": 508.63, "stderr": 1.494, "cost_per_test": 0.190917, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 74.523, "latency": 379.243, "stderr": 1.44, "cost_per_test": 0.013134, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder-full": { "accuracy": 74.094, "latency": 1026.858, "stderr": 1.572, "cost_per_test": 0.444098, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 74.034, "latency": 353.159, "stderr": 1.502, "cost_per_test": 0.127988, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 73.913, "latency": 311.74, "stderr": 1.548, "cost_per_test": 0.237214, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 73.747, "latency": 337.699, "stderr": 1.457, "cost_per_test": 0.094398, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 73.744, "latency": 191.188, "stderr": 1.549, "cost_per_test": 0.05932, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 73.278, "latency": 250.64, "stderr": 1.555, "cost_per_test": 0.320778, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 72.528, "latency": 288.391, "stderr": 1.471, "cost_per_test": 0.15542, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 72.15, "latency": 367.969, "stderr": 1.569, "cost_per_test": 0.015189, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 71.19, "latency": 192.151, "stderr": 1.535, "cost_per_test": 0.070122, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 70.918, "latency": 322.4, "stderr": 1.54, "cost_per_test": 0.307623, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 70.619, "latency": 177.751, "stderr": 1.675, "cost_per_test": 0.085966, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 69.949, "latency": 670.02, "stderr": 1.598, "cost_per_test": 0.295262, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 69.46, "latency": 220.97, "stderr": 1.656, "cost_per_test": 0.023081, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 68.876, "latency": 753.026, "stderr": 1.688, "cost_per_test": 1.72152, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 68.156, "latency": 169.099, "stderr": 1.584, "cost_per_test": 0.026765, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 67.296, "latency": 349.074, "stderr": 1.682, "cost_per_test": 0.020924, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 66.31, "latency": 443.204, "stderr": 1.502, "cost_per_test": 0.122276, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 65.907, "latency": 115.204, "stderr": 1.577, "cost_per_test": 0.09976, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 65.774, "latency": 136.024, "stderr": 1.565, "cost_per_test": 0.100547, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 65.197, "latency": 411.312, "stderr": 1.717, "cost_per_test": 0.233012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 62.253, "latency": 140.018, "stderr": 1.657, "cost_per_test": 0.128701, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 58.103, "latency": 94.051, "stderr": 1.699, "cost_per_test": 0.025801, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "elicitation": { "meta/muse_spark_1_3_max": { "accuracy": 69.1, "latency": 177.751, "stderr": 1.722, "cost_per_test": 0.085966, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 69.087, "latency": 192.151, "stderr": 1.769, "cost_per_test": 0.070122, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 68.226, "latency": 191.188, "stderr": 1.802, "cost_per_test": 0.05932, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 67.03, "latency": 250.64, "stderr": 1.795, "cost_per_test": 0.320778, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 65.986, "latency": 1026.858, "stderr": 1.771, "cost_per_test": 0.444098, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 63.761, "latency": 288.391, "stderr": 1.875, "cost_per_test": 0.15542, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 63.103, "latency": 525.253, "stderr": 1.954, "cost_per_test": 0.80344, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 62.92, "latency": 753.026, "stderr": 1.924, "cost_per_test": 1.72152, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 61.852, "latency": 369.608, "stderr": 2.083, "cost_per_test": 0.469621, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 61.124, "latency": 322.4, "stderr": 2.05, "cost_per_test": 0.307623, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 60.62, "latency": 353.159, "stderr": 1.995, "cost_per_test": 0.127988, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 59.536, "latency": 670.02, "stderr": 1.976, "cost_per_test": 0.295262, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 59.097, "latency": 379.243, "stderr": 1.984, "cost_per_test": 0.013134, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 58.459, "latency": 652.03, "stderr": 1.963, "cost_per_test": 1.633943, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 57.661, "latency": 367.969, "stderr": 2.098, "cost_per_test": 0.015189, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 55.857, "latency": 411.312, "stderr": 2.145, "cost_per_test": 0.233012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 54.368, "latency": 220.97, "stderr": 2.202, "cost_per_test": 0.023081, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 54.194, "latency": 337.699, "stderr": 2.122, "cost_per_test": 0.094398, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 53.998, "latency": 115.204, "stderr": 2.065, "cost_per_test": 0.09976, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 53.843, "latency": 169.099, "stderr": 2.13, "cost_per_test": 0.026765, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 53.267, "latency": 349.074, "stderr": 2.152, "cost_per_test": 0.020924, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 53.217, "latency": 348.037, "stderr": 2.101, "cost_per_test": 1.14488, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 51.738, "latency": 311.74, "stderr": 2.221, "cost_per_test": 0.237214, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 51.274, "latency": 94.051, "stderr": 2.067, "cost_per_test": 0.025801, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 49.827, "latency": 443.204, "stderr": 2.093, "cost_per_test": 0.122276, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 49.719, "latency": 140.018, "stderr": 2.038, "cost_per_test": 0.128701, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 49.588, "latency": 508.63, "stderr": 2.106, "cost_per_test": 0.190917, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 48.667, "latency": 136.024, "stderr": 2.072, "cost_per_test": 0.100547, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } } } }