{ "metadata": { "benchmark": "Public Benefits Bench v1", "slug": "public-benefits-bench-v1", "description": "Can AI help people navigate SNAP benefits?", "benchmark_id": "snap_v1", "family": "snap", "version": "1", "updated": "2026-06-09", "dataset_type": "private", "industry": "social mobility", "tasks": { "overall": "Overall", "both": "Both", "web_search": "Web Search", "multi_turn": "Multi Turn", "neither": "Neither" }, "models": [ "anthropic/claude-fable-5", "anthropic/claude-haiku-4-5-20251001", "anthropic/claude-opus-4-8", "anthropic/claude-sonnet-4-6", "deepseek/deepseek-v4-pro", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "grok/grok-4-1-fast-reasoning", "grok/grok-4.3", "kimi/kimi-k2.6", "minimax/MiniMax-M3", "openai/gpt-5.5", "zai/glm-5.1" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": true, "total_models": 13 }, "tasks": { "overall": { "anthropic/claude-fable-5": { "accuracy": 71.651, "latency": 1038.337, "stderr": 1.172, "cost_per_test": 1.054998, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 62.111, "latency": 597.691, "stderr": 1.262, "cost_per_test": 1.316771, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 60.69, "latency": 705.544, "stderr": 1.27, "cost_per_test": 0.198621, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 58.525, "latency": 787.612, "stderr": 1.282, "cost_per_test": 0.816309, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 57.984, "latency": 273.985, "stderr": 1.284, "cost_per_test": 0.775836, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 57.916, "latency": 437.877, "stderr": 1.284, "cost_per_test": 0.339125, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 57.578, "latency": 977.871, "stderr": 1.286, "cost_per_test": 0.359673, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 57.24, "latency": 1580.527, "stderr": 1.287, "cost_per_test": 2.018596, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 53.383, "latency": 578.919, "stderr": 1.298, "cost_per_test": 0.611734, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 50.88, "latency": 177.913, "stderr": 1.3, "cost_per_test": 0.300864, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 50.068, "latency": 418.342, "stderr": 1.301, "cost_per_test": 1.207438, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 49.526, "latency": 190.393, "stderr": 1.301, "cost_per_test": 0.194641, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 44.317, "latency": 185.115, "stderr": 1.292, "cost_per_test": 0.064873, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "both": { "anthropic/claude-fable-5": { "accuracy": 71.651, "latency": 1038.337, "stderr": 1.172, "cost_per_test": 1.054998, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 62.111, "latency": 597.691, "stderr": 1.262, "cost_per_test": 1.316771, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 60.69, "latency": 705.544, "stderr": 1.27, "cost_per_test": 0.198621, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 58.525, "latency": 787.612, "stderr": 1.282, "cost_per_test": 0.816309, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 57.984, "latency": 273.985, "stderr": 1.284, "cost_per_test": 0.775836, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 57.916, "latency": 437.877, "stderr": 1.284, "cost_per_test": 0.339125, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 57.578, "latency": 977.871, "stderr": 1.286, "cost_per_test": 0.359673, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 57.24, "latency": 1580.527, "stderr": 1.287, "cost_per_test": 2.018596, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 53.383, "latency": 578.919, "stderr": 1.298, "cost_per_test": 0.611734, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 50.88, "latency": 177.913, "stderr": 1.3, "cost_per_test": 0.300864, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 50.068, "latency": 418.342, "stderr": 1.301, "cost_per_test": 1.207438, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 49.526, "latency": 190.393, "stderr": 1.301, "cost_per_test": 0.194641, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 44.317, "latency": 185.115, "stderr": 1.292, "cost_per_test": 0.064873, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "web_search": { "anthropic/claude-fable-5": { "accuracy": 67.727, "latency": 792.684, "stderr": 1.216, "cost_per_test": 0.729545, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 58.931, "latency": 397.297, "stderr": 1.28, "cost_per_test": 0.749876, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 54.127, "latency": 155.22, "stderr": 1.296, "cost_per_test": 0.603029, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 53.992, "latency": 658.98, "stderr": 1.296, "cost_per_test": 0.212826, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 53.857, "latency": 1325.885, "stderr": 1.297, "cost_per_test": 1.301837, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 53.586, "latency": 436.032, "stderr": 1.297, "cost_per_test": 0.078653, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 52.368, "latency": 485.702, "stderr": 1.299, "cost_per_test": 0.418844, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 50.744, "latency": 190.702, "stderr": 1.3, "cost_per_test": 0.114843, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 44.046, "latency": 105.689, "stderr": 1.291, "cost_per_test": 0.065444, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 43.572, "latency": 180.653, "stderr": 1.29, "cost_per_test": 0.209478, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 41.813, "latency": 73.762, "stderr": 1.283, "cost_per_test": 0.143993, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 36.13, "latency": 67.591, "stderr": 1.25, "cost_per_test": 0.067391, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 33.085, "latency": 31.551, "stderr": 1.224, "cost_per_test": 0.004028, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "multi_turn": { "anthropic/claude-fable-5": { "accuracy": 65.223, "latency": 272.419, "stderr": 1.239, "cost_per_test": 0.294256, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 58.457, "latency": 311.358, "stderr": 1.282, "cost_per_test": 0.592054, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 53.992, "latency": 129.707, "stderr": 1.296, "cost_per_test": 0.130219, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 53.857, "latency": 434.054, "stderr": 1.297, "cost_per_test": 0.567811, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 52.909, "latency": 109.306, "stderr": 1.298, "cost_per_test": 0.087001, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 52.368, "latency": 363.464, "stderr": 1.299, "cost_per_test": 0.278287, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 48.714, "latency": 350.63, "stderr": 1.3, "cost_per_test": 0.086444, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.241, "latency": 492.82, "stderr": 1.3, "cost_per_test": 0.086954, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.1": { "accuracy": 47.564, "latency": 186.749, "stderr": 1.299, "cost_per_test": 0.024678, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 46.888, "latency": 155.65, "stderr": 1.298, "cost_per_test": 0.049242, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 46.346, "latency": 214.475, "stderr": 1.297, "cost_per_test": 0.029972, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 41.678, "latency": 110.415, "stderr": 1.282, "cost_per_test": 0.005668, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 33.897, "latency": 143.916, "stderr": 1.231, "cost_per_test": 0.033656, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null } }, "neither": { "anthropic/claude-fable-5": { "accuracy": 63.396, "latency": 143.58, "stderr": 1.253, "cost_per_test": 0.145973, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 53.112, "latency": 148.613, "stderr": 1.298, "cost_per_test": 0.269121, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 50.271, "latency": 54.325, "stderr": 1.301, "cost_per_test": 0.062674, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 49.729, "latency": 224.052, "stderr": 1.301, "cost_per_test": 0.257766, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 47.429, "latency": 34.272, "stderr": 1.299, "cost_per_test": 0.032702, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 45.602, "latency": 137.81, "stderr": 1.296, "cost_per_test": 0.105211, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 43.234, "latency": 143.183, "stderr": 1.289, "cost_per_test": 0.027685, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 40.189, "latency": 105.499, "stderr": 1.275, "cost_per_test": 0.010008, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.3": { "accuracy": 37.957, "latency": 21.776, "stderr": 1.262, "cost_per_test": 0.006594, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.1": { "accuracy": 36.198, "latency": 38.926, "stderr": 1.25, "cost_per_test": 0.004344, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 34.438, "latency": 23.517, "stderr": 1.236, "cost_per_test": 0.004717, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 33.018, "latency": 15.402, "stderr": 1.223, "cost_per_test": 0.001004, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 22.598, "latency": 21.037, "stderr": 1.088, "cost_per_test": 0.005832, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null } } } }