{ "metadata": { "benchmark": "Public Benefits Bench v1.1", "slug": "public-benefits-bench", "description": "Can AI help people navigate SNAP benefits?", "benchmark_id": "snap", "family": "snap", "version": "1.1", "updated": "2026-10-12", "dataset_type": "private", "industry": "social mobility", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4.1-flash", "fireworks/ember-1", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4-1-fast-reasoning", "grok/grok-4.3", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M3", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "poolside/laguna-m.1", "poolside/laguna-xs.2", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.1", "zai/glm-5.3" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "latency_label": "Target-agent time", "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 50 }, "tasks": { "overall": { "anthropic/claude-opus-5": { "accuracy": 76.928, "latency": 1808.881, "stderr": 1.096, "cost_per_test": 3.949099, "token_totals": { "input_tokens": 704875690, "output_tokens": 12509272, "reasoning_tokens": 6498299, "cache_read_tokens": 662589765, "cache_write_tokens": 42269235 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 74.899, "latency": 2882.401, "stderr": 1.128, "cost_per_test": 7.481573, "token_totals": { "input_tokens": 764685044, "output_tokens": 11288197, "reasoning_tokens": 5916781, "cache_read_tokens": 685891147, "cache_write_tokens": 78776117 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 73.748, "latency": 4364.892, "stderr": 1.145, "cost_per_test": 4.635817, "token_totals": { "input_tokens": 1498277154, "output_tokens": 9571943, "reasoning_tokens": 3995930, "cache_read_tokens": 1346923600, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.636, "latency": 4771.81, "stderr": 1.185, "cost_per_test": 5.952407, "token_totals": { "input_tokens": 906177236, "output_tokens": 18318977, "reasoning_tokens": 14012794, "cache_read_tokens": 766997389, "cache_write_tokens": 139155673 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 70.433, "latency": 1276.576, "stderr": 1.187, "cost_per_test": 4.585446, "token_totals": { "input_tokens": 263243176, "output_tokens": 9748421, "reasoning_tokens": null, "cache_read_tokens": 236807281, "cache_write_tokens": 26426081 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 70.23, "latency": 1184.49, "stderr": 1.189, "cost_per_test": 2.457181, "token_totals": { "input_tokens": 1156134294, "output_tokens": 7333084, "reasoning_tokens": 3034066, "cache_read_tokens": 1106854548, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 69.756, "latency": 1566.785, "stderr": 0.0, "cost_per_test": 2.793902, "token_totals": { "input_tokens": 658538120, "output_tokens": 6776808, "reasoning_tokens": 3500703, "cache_read_tokens": 559760847, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 68.945, "latency": 2234.344, "stderr": 1.204, "cost_per_test": 0.081936, "token_totals": { "input_tokens": 322998308, "output_tokens": 10223982, "reasoning_tokens": 7494264, "cache_read_tokens": 302628352, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 68.606, "latency": 3755.023, "stderr": 1.207, "cost_per_test": 0.252532, "token_totals": { "input_tokens": 380517580, "output_tokens": 8139637, "reasoning_tokens": 5733788, "cache_read_tokens": 353063872, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 68.539, "latency": 2150.623, "stderr": 1.208, "cost_per_test": 0.946832, "token_totals": { "input_tokens": 409815704, "output_tokens": 12440521, "reasoning_tokens": null, "cache_read_tokens": 360270907, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 68.471, "latency": 535.162, "stderr": 1.209, "cost_per_test": 0.587608, "token_totals": { "input_tokens": 388336719, "output_tokens": 5740007, "reasoning_tokens": 1080003, "cache_read_tokens": 340605576, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 68.2, "latency": 438.672, "stderr": 1.211, "cost_per_test": 0.913402, "token_totals": { "input_tokens": 255356799, "output_tokens": 6007238, "reasoning_tokens": 3763309, "cache_read_tokens": 239294974, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 68.133, "latency": 767.85, "stderr": 1.212, "cost_per_test": 1.889795, "token_totals": { "input_tokens": 152942625, "output_tokens": 9270173, "reasoning_tokens": null, "cache_read_tokens": 130953865, "cache_write_tokens": 21982282 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 67.591, "latency": 1255.522, "stderr": 1.217, "cost_per_test": 0.031798, "token_totals": { "input_tokens": 357795247, "output_tokens": 9949392, "reasoning_tokens": 6091752, "cache_read_tokens": 332096768, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 67.185, "latency": 4527.815, "stderr": 1.221, "cost_per_test": 4.245484, "token_totals": { "input_tokens": 1294364352, "output_tokens": 29003787, "reasoning_tokens": 23360865, "cache_read_tokens": 1108467518, "cache_write_tokens": 185872526 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.118, "latency": 2443.976, "stderr": 1.222, "cost_per_test": 0.991386, "token_totals": { "input_tokens": 506841122, "output_tokens": 8290569, "reasoning_tokens": 5695192, "cache_read_tokens": 477375360, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 66.847, "latency": 1552.915, "stderr": 1.225, "cost_per_test": 0.91559, "token_totals": { "input_tokens": 275022880, "output_tokens": 3184521, "reasoning_tokens": 706129, "cache_read_tokens": 239044736, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 66.509, "latency": 3677.634, "stderr": 1.228, "cost_per_test": 8.847882, "token_totals": { "input_tokens": 1506316388, "output_tokens": 8001721, "reasoning_tokens": 4848671, "cache_read_tokens": 1350622720, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 66.171, "latency": 801.415, "stderr": 1.231, "cost_per_test": 1.023263, "token_totals": { "input_tokens": 307586574, "output_tokens": 4726999, "reasoning_tokens": 2652878, "cache_read_tokens": 280857078, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 66.035, "latency": 1268.852, "stderr": 1.232, "cost_per_test": 1.290695, "token_totals": { "input_tokens": 373274847, "output_tokens": 12699538, "reasoning_tokens": null, "cache_read_tokens": 331877261, "cache_write_tokens": 41387480 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 65.629, "latency": 1016.967, "stderr": 1.235, "cost_per_test": 2.246775, "token_totals": { "input_tokens": 606356401, "output_tokens": 6924993, "reasoning_tokens": 3893035, "cache_read_tokens": 491669632, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 65.291, "latency": 201.419, "stderr": 1.238, "cost_per_test": 1.011271, "token_totals": { "input_tokens": 539398130, "output_tokens": 5287711, "reasoning_tokens": 3569491, "cache_read_tokens": 456416879, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 64.276, "latency": 598.96, "stderr": 1.246, "cost_per_test": 0.073234, "token_totals": { "input_tokens": 324083542, "output_tokens": 7830427, "reasoning_tokens": 5092304, "cache_read_tokens": 305366398, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 64.141, "latency": 691.436, "stderr": 1.247, "cost_per_test": 0.2543, "token_totals": { "input_tokens": 284714117, "output_tokens": 4543318, "reasoning_tokens": null, "cache_read_tokens": 256756972, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 63.194, "latency": 2192.484, "stderr": 1.254, "cost_per_test": 0.525402, "token_totals": { "input_tokens": 521915572, "output_tokens": 37160233, "reasoning_tokens": 32909550, "cache_read_tokens": 443798683, "cache_write_tokens": 78101633 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 62.923, "latency": 922.395, "stderr": 1.256, "cost_per_test": 0.460453, "token_totals": { "input_tokens": 416776131, "output_tokens": 3598874, "reasoning_tokens": 1104352, "cache_read_tokens": 396119296, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 62.449, "latency": 882.995, "stderr": 1.26, "cost_per_test": 1.105228, "token_totals": { "input_tokens": 308948823, "output_tokens": 5727806, "reasoning_tokens": null, "cache_read_tokens": 287033952, "cache_write_tokens": 21907318 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 62.382, "latency": 1171.486, "stderr": 1.26, "cost_per_test": 1.199723, "token_totals": { "input_tokens": 641043129, "output_tokens": 5293935, "reasoning_tokens": 2859562, "cache_read_tokens": 594265088, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 61.84, "latency": 580.381, "stderr": 1.264, "cost_per_test": 0.340666, "token_totals": { "input_tokens": 253316316, "output_tokens": 2189961, "reasoning_tokens": 0, "cache_read_tokens": 227463680, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 61.299, "latency": 395.054, "stderr": 1.267, "cost_per_test": 0.278428, "token_totals": { "input_tokens": 213360122, "output_tokens": 4024960, "reasoning_tokens": 929960, "cache_read_tokens": 199788960, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 61.164, "latency": 2319.748, "stderr": 1.268, "cost_per_test": 0.270182, "token_totals": { "input_tokens": 1526164237, "output_tokens": 8585020, "reasoning_tokens": 5566100, "cache_read_tokens": 1407739326, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 60.893, "latency": 2377.308, "stderr": 1.269, "cost_per_test": 3.971165, "token_totals": { "input_tokens": 859486603, "output_tokens": 5799841, "reasoning_tokens": 3539368, "cache_read_tokens": 790680064, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 59.472, "latency": 338.872, "stderr": 1.277, "cost_per_test": 1.113099, "token_totals": { "input_tokens": 574078082, "output_tokens": 6172733, "reasoning_tokens": 4296597, "cache_read_tokens": 489377006, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 59.405, "latency": 521.336, "stderr": 1.277, "cost_per_test": 0.095937, "token_totals": { "input_tokens": 221724908, "output_tokens": 4561326, "reasoning_tokens": null, "cache_read_tokens": 208023296, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 59.337, "latency": 3024.455, "stderr": 1.278, "cost_per_test": 0.690795, "token_totals": { "input_tokens": 315084952, "output_tokens": 7684981, "reasoning_tokens": 5458507, "cache_read_tokens": 294009060, "cache_write_tokens": 20960904 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 58.457, "latency": 810.832, "stderr": 1.282, "cost_per_test": 0.367636, "token_totals": { "input_tokens": 311973011, "output_tokens": 4537151, "reasoning_tokens": null, "cache_read_tokens": 296135168, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "stepfun/step-5-preview": { "accuracy": 58.119, "latency": 1639.157, "stderr": 1.283, "cost_per_test": 0.504407, "token_totals": { "input_tokens": 756809486, "output_tokens": 12008540, "reasoning_tokens": 0, "cache_read_tokens": 708651520, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 57.645, "latency": 3178.908, "stderr": 1.285, "cost_per_test": 0.224168, "token_totals": { "input_tokens": 1825774915, "output_tokens": 17789521, "reasoning_tokens": 13913959, "cache_read_tokens": 1725599246, "cache_write_tokens": 99654947 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 56.631, "latency": 595.296, "stderr": 1.289, "cost_per_test": 0.906347, "token_totals": { "input_tokens": 979048945, "output_tokens": 2255010, "reasoning_tokens": 0, "cache_read_tokens": 924881920, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 56.631, "latency": 1862.494, "stderr": 1.289, "cost_per_test": 2.491613, "token_totals": { "input_tokens": 1552257632, "output_tokens": 7014313, "reasoning_tokens": 3446365, "cache_read_tokens": 1494255406, "cache_write_tokens": 57401341 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 56.563, "latency": 121.522, "stderr": 1.289, "cost_per_test": 0.385496, "token_totals": { "input_tokens": 119300944, "output_tokens": 3282758, "reasoning_tokens": 2147005, "cache_read_tokens": 85117031, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 54.263, "latency": 177.657, "stderr": 1.296, "cost_per_test": 0.224512, "token_totals": { "input_tokens": 175571132, "output_tokens": 2567725, "reasoning_tokens": null, "cache_read_tokens": 156989018, "cache_write_tokens": 18072905 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 53.789, "latency": 185.553, "stderr": 1.297, "cost_per_test": 0.376768, "token_totals": { "input_tokens": 77067942, "output_tokens": 2396732, "reasoning_tokens": 1547863, "cache_read_tokens": 53466726, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 51.827, "latency": 69.26, "stderr": 1.3, "cost_per_test": 0.09565, "token_totals": { "input_tokens": 84048754, "output_tokens": 1957055, "reasoning_tokens": 1249909, "cache_read_tokens": 30028794, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 51.691, "latency": 205.01, "stderr": 1.3, "cost_per_test": 0.264337, "token_totals": { "input_tokens": 120067770, "output_tokens": 3434983, "reasoning_tokens": 1482191, "cache_read_tokens": 93465618, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 50.744, "latency": 275.254, "stderr": 1.3, "cost_per_test": 0.049736, "token_totals": { "input_tokens": 434280369, "output_tokens": 4656514, "reasoning_tokens": null, "cache_read_tokens": 369269791, "cache_write_tokens": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "inception/mercury-2.5": { "accuracy": 45.264, "latency": 68.325, "stderr": 1.295, "cost_per_test": 0.067576, "token_totals": { "input_tokens": 227432781, "output_tokens": 2367427, "reasoning_tokens": 1625567, "cache_read_tokens": 176220251, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 44.79, "latency": 116.823, "stderr": 1.293, "cost_per_test": 0.031221, "token_totals": { "input_tokens": 69903464, "output_tokens": 2373913, "reasoning_tokens": 722305, "cache_read_tokens": 53245958, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 43.843, "latency": 334.832, "stderr": 1.291, "cost_per_test": 0.201829, "token_totals": { "input_tokens": 410949945, "output_tokens": 2031370, "reasoning_tokens": 485323, "cache_read_tokens": 365818068, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 41.407, "latency": 305.41, "stderr": 1.281, "cost_per_test": 0.113029, "token_totals": { "input_tokens": 415335868, "output_tokens": 1934540, "reasoning_tokens": 591448, "cache_read_tokens": 318477696, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } } } }