{ "metadata": { "benchmark": "Vals Science Index", "slug": "science_index", "description": "A single measure of AI's ability to do science \u2014 agentic model performance on biology data investigation, mechanism discovery, and research workflows, weighted equally.", "benchmark_id": "science_index", "family": "science_index", "version": "1.0", "updated": "2026-10-16", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "biomysterybench": "BioMysteryBench", "mysterymechanism": "MysteryMechanism", "terminal_bench_science": "Terminal-Bench-Science" }, "models": [ "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4.1-flash", "fireworks/ember-1", "google/gemini-3.1-pro-preview", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.6", "grok/grok-4.7", "meta/muse-wonder", "meta/muse-wonder-full", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "stepfun/step-5-preview", "xiaomi/mimo-v2.6-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 21 }, "tasks": { "overall": { "openai/gpt-6-astra": { "accuracy": 65.09, "latency": 2436.537, "stderr": 2.262, "cost_per_test": 8.027277, "token_totals": { "input_tokens": 884863011, "output_tokens": 25809418, "reasoning_tokens": 12943157, "cache_read_tokens": 449914908, "cache_write_tokens": 25072593 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 59.628, "latency": 1990.571, "stderr": 2.307, "cost_per_test": 1.364659, "token_totals": { "input_tokens": 757076271, "output_tokens": 27121303, "reasoning_tokens": 13763480, "cache_read_tokens": 388775477, "cache_write_tokens": 30189914 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 59.118, "latency": 4158.815, "stderr": 2.305, "cost_per_test": 13.235304, "token_totals": { "input_tokens": 20046719376, "output_tokens": 115887424, "reasoning_tokens": 67740975, "cache_read_tokens": 10815790916, "cache_write_tokens": 119754753 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 58.651, "latency": 3591.274, "stderr": 2.318, "cost_per_test": 9.12802, "token_totals": { "input_tokens": 4676300983, "output_tokens": 64760814, "reasoning_tokens": 39539150, "cache_read_tokens": 2669242789, "cache_write_tokens": 177708111 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 55.359, "latency": 3420.286, "stderr": 2.288, "cost_per_test": 18.402067, "token_totals": { "input_tokens": 4955353347, "output_tokens": 70137813, "reasoning_tokens": 32538408, "cache_read_tokens": 2578283096, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 48.537, "latency": 5662.568, "stderr": 2.225, "cost_per_test": 10.150213, "token_totals": { "input_tokens": 5132642295, "output_tokens": 45144071, "reasoning_tokens": 29757091, "cache_read_tokens": 4891569963, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 47.93, "latency": 3705.653, "stderr": 2.199, "cost_per_test": 13.038951, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 44.998, "latency": 1876.106, "stderr": 2.258, "cost_per_test": 2.314922, "token_totals": { "input_tokens": 2416077678, "output_tokens": 23614102, "reasoning_tokens": 12283114, "cache_read_tokens": 1280231223, "cache_write_tokens": 29006459 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 41.481, "latency": 2612.344, "stderr": 1.934, "cost_per_test": 3.403486, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 39.124, "latency": 2587.679, "stderr": 1.964, "cost_per_test": 8.742674, "token_totals": { "input_tokens": 5899322456, "output_tokens": 45376332, "reasoning_tokens": 29190884, "cache_read_tokens": 5810889987, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 38.502, "latency": 2398.32, "stderr": 1.671, "cost_per_test": 1.859159, "token_totals": { "input_tokens": 11080720563, "output_tokens": 103357352, "reasoning_tokens": 67306753, "cache_read_tokens": 6010191105, "cache_write_tokens": 105764540 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 38.094, "latency": 2409.528, "stderr": 1.643, "cost_per_test": 2.450732, "token_totals": { "input_tokens": 1281975686, "output_tokens": 38705909, "reasoning_tokens": 23972346, "cache_read_tokens": 695373781, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 35.76, "latency": 1320.142, "stderr": 1.657, "cost_per_test": 2.975983, "token_totals": { "input_tokens": 3843336750, "output_tokens": 38761438, "reasoning_tokens": 17671298, "cache_read_tokens": 2360671595, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 34.828, "latency": 2434.191, "stderr": 1.611, "cost_per_test": 7.123405, "token_totals": { "input_tokens": 2840485778, "output_tokens": 40435603, "reasoning_tokens": 18014691, "cache_read_tokens": 1558762453, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 31.758, "latency": 4659.396, "stderr": 1.663, "cost_per_test": 10.987134, "token_totals": { "input_tokens": 11349286979, "output_tokens": 48805252, "reasoning_tokens": 25397188, "cache_read_tokens": 5752309510, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 31.722, "latency": 6047.487, "stderr": 1.272, "cost_per_test": 0.117321, "token_totals": { "input_tokens": 4918806101, "output_tokens": 66879818, "reasoning_tokens": 40551870, "cache_read_tokens": 2752730944, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 29.65, "latency": 3998.656, "stderr": 0.988, "cost_per_test": 0.290142, "token_totals": { "input_tokens": 4412167254, "output_tokens": 70921915, "reasoning_tokens": 46020082, "cache_read_tokens": 2635955320, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 29.577, "latency": 4017.643, "stderr": 1.104, "cost_per_test": 1.780912, "token_totals": { "input_tokens": 8892898547, "output_tokens": 56528893, "reasoning_tokens": 0, "cache_read_tokens": 4742501376, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 28.379, "latency": 1814.828, "stderr": 1.481, "cost_per_test": 0.106685, "token_totals": { "input_tokens": 1766018605, "output_tokens": 29983661, "reasoning_tokens": 17799973, "cache_read_tokens": 961803372, "cache_write_tokens": 32735080 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 27.383, "latency": 1083.463, "stderr": 1.419, "cost_per_test": 2.561873, "token_totals": { "input_tokens": 1992466946, "output_tokens": 16329518, "reasoning_tokens": 6920343, "cache_read_tokens": 1074011989, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 25.298, "latency": 3000.259, "stderr": 0.825, "cost_per_test": 0.259854, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "biomysterybench": { "anthropic/claude-sonnet-5-5": { "accuracy": 81.111, "latency": 1086.061, "stderr": 0.642, "cost_per_test": 3.736494, "token_totals": { "input_tokens": 732589448, "output_tokens": 14342557, "reasoning_tokens": 7945289, "cache_read_tokens": 712471480, "cache_write_tokens": 20094500 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 79.63, "latency": 530.7, "stderr": 0.741, "cost_per_test": 0.372147, "token_totals": { "input_tokens": 49187181, "output_tokens": 1921230, "reasoning_tokens": 1213195, "cache_read_tokens": 45054185, "cache_write_tokens": 4050774 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 79.259, "latency": 286.147, "stderr": 0.98, "cost_per_test": 1.719326, "token_totals": { "input_tokens": 39863504, "output_tokens": 1463159, "reasoning_tokens": 815272, "cache_read_tokens": 35228179, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 79.259, "latency": 739.252, "stderr": 0.98, "cost_per_test": 3.642341, "token_totals": { "input_tokens": 359126168, "output_tokens": 6342038, "reasoning_tokens": 4159050, "cache_read_tokens": 348450323, "cache_write_tokens": 10652845 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 79.259, "latency": 2195.964, "stderr": 2.062, "cost_per_test": 3.292774, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 76.296, "latency": 2396.816, "stderr": 0.37, "cost_per_test": 5.707134, "token_totals": { "input_tokens": 187578219, "output_tokens": 16476596, "reasoning_tokens": 11274521, "cache_read_tokens": 149000710, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 74.815, "latency": 324.465, "stderr": 2.429, "cost_per_test": 0.662358, "token_totals": { "input_tokens": 129632435, "output_tokens": 2217483, "reasoning_tokens": 1401831, "cache_read_tokens": 124856548, "cache_write_tokens": 4671549 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 73.333, "latency": 1574.165, "stderr": 0.642, "cost_per_test": 3.473154, "token_totals": { "input_tokens": 712304421, "output_tokens": 5992307, "reasoning_tokens": 3536726, "cache_read_tokens": 696010395, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 72.222, "latency": 858.019, "stderr": 0.0, "cost_per_test": 1.226606, "token_totals": { "input_tokens": 104073191, "output_tokens": 4208045, "reasoning_tokens": 2899195, "cache_read_tokens": 82000085, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 71.111, "latency": 1098.261, "stderr": 0.642, "cost_per_test": 1.925036, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 70.37, "latency": 752.457, "stderr": 1.335, "cost_per_test": 0.512756, "token_totals": { "input_tokens": 241182764, "output_tokens": 16695865, "reasoning_tokens": 14393004, "cache_read_tokens": 217680976, "cache_write_tokens": 23494403 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 69.259, "latency": 696.818, "stderr": 1.335, "cost_per_test": 2.487742, "token_totals": { "input_tokens": 254511912, "output_tokens": 3699044, "reasoning_tokens": 2064454, "cache_read_tokens": 215793109, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 69.259, "latency": 1707.953, "stderr": 0.98, "cost_per_test": 0.050119, "token_totals": { "input_tokens": 363783190, "output_tokens": 6591713, "reasoning_tokens": 4865655, "cache_read_tokens": 351782592, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 67.778, "latency": 890.067, "stderr": 1.111, "cost_per_test": 0.233183, "token_totals": { "input_tokens": 268679692, "output_tokens": 14743987, "reasoning_tokens": 13033344, "cache_read_tokens": 262450176, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 67.407, "latency": 1294.751, "stderr": 0.37, "cost_per_test": 0.494625, "token_totals": { "input_tokens": 270017689, "output_tokens": 7012589, "reasoning_tokens": 0, "cache_read_tokens": 257300480, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "fireworks/ember-1": { "accuracy": 67.037, "latency": 566.167, "stderr": 3.032, "cost_per_test": 0.907066, "token_totals": { "input_tokens": 127490411, "output_tokens": 1806362, "reasoning_tokens": 999895, "cache_read_tokens": 121455841, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder": { "accuracy": 67.037, "latency": 1081.905, "stderr": 0.37, "cost_per_test": 3.228928, "token_totals": { "input_tokens": 656406298, "output_tokens": 4876338, "reasoning_tokens": 2954517, "cache_read_tokens": 638118607, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 62.222, "latency": 326.755, "stderr": 1.697, "cost_per_test": 1.5539, "token_totals": { "input_tokens": 391907840, "output_tokens": 3890918, "reasoning_tokens": 1948190, "cache_read_tokens": 355089274, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 61.481, "latency": 523.805, "stderr": 2.593, "cost_per_test": 0.053236, "token_totals": { "input_tokens": 157508494, "output_tokens": 4482770, "reasoning_tokens": 3258485, "cache_read_tokens": 149567191, "cache_write_tokens": 7787108 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 61.481, "latency": 1111.274, "stderr": 0.741, "cost_per_test": 0.088859, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 60.0, "latency": 251.867, "stderr": 2.94, "cost_per_test": 0.713378, "token_totals": { "input_tokens": 74631111, "output_tokens": 2138194, "reasoning_tokens": 1319799, "cache_read_tokens": 62626251, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "mysterymechanism": { "openai/gpt-6-astra": { "accuracy": 53.153, "latency": 488.597, "stderr": 3.357, "cost_per_test": 1.559704, "token_totals": { "input_tokens": 34970642, "output_tokens": 4706731, "reasoning_tokens": 3583655, "cache_read_tokens": 28320058, "cache_write_tokens": 6436778 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 49.55, "latency": 1178.454, "stderr": 3.363, "cost_per_test": 4.619754, "token_totals": { "input_tokens": 658569339, "output_tokens": 29182250, "reasoning_tokens": 23537093, "cache_read_tokens": 619837485, "cache_write_tokens": 38715822 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 49.099, "latency": 1410.236, "stderr": 3.363, "cost_per_test": 3.45323, "token_totals": { "input_tokens": 1091907970, "output_tokens": 42938841, "reasoning_tokens": 35532465, "cache_read_tokens": 1040231244, "cache_write_tokens": 51657746 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 46.396, "latency": 437.025, "stderr": 3.355, "cost_per_test": 0.284881, "token_totals": { "input_tokens": 32268619, "output_tokens": 4481195, "reasoning_tokens": 3364816, "cache_read_tokens": 25888801, "cache_write_tokens": 6166336 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 45.495, "latency": 1494.947, "stderr": 3.35, "cost_per_test": 4.594426, "token_totals": { "input_tokens": 901582710, "output_tokens": 21508801, "reasoning_tokens": 10866656, "cache_read_tokens": 793827453, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 37.387, "latency": 1012.543, "stderr": 3.255, "cost_per_test": 3.280693, "token_totals": { "input_tokens": 260014806, "output_tokens": 18050903, "reasoning_tokens": 13538433, "cache_read_tokens": 234441635, "cache_write_tokens": 25563677 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 36.486, "latency": 322.237, "stderr": 3.238, "cost_per_test": 1.731041, "token_totals": { "input_tokens": 827206914, "output_tokens": 17858080, "reasoning_tokens": 9860899, "cache_read_tokens": 733670345, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 35.135, "latency": 805.031, "stderr": 3.211, "cost_per_test": 0.541024, "token_totals": { "input_tokens": 910086657, "output_tokens": 41490334, "reasoning_tokens": 33895469, "cache_read_tokens": 861339525, "cache_write_tokens": 48729800 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 35.135, "latency": 2494.329, "stderr": 3.211, "cost_per_test": 4.163398, "token_totals": { "input_tokens": 707253940, "output_tokens": 22089086, "reasoning_tokens": 14663498, "cache_read_tokens": 617506072, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 33.333, "latency": 522.744, "stderr": 3.171, "cost_per_test": 0.96335, "token_totals": { "input_tokens": 130077211, "output_tokens": 6106439, "reasoning_tokens": 3933893, "cache_read_tokens": 121397470, "cache_write_tokens": 8456854 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 30.631, "latency": 1674.514, "stderr": 3.101, "cost_per_test": 1.309658, "token_totals": { "input_tokens": 161102384, "output_tokens": 23854775, "reasoning_tokens": 17588077, "cache_read_tokens": 116392832, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 30.18, "latency": 373.426, "stderr": 3.088, "cost_per_test": 0.460931, "token_totals": { "input_tokens": 73446321, "output_tokens": 6771421, "reasoning_tokens": 5398618, "cache_read_tokens": 64736689, "cache_write_tokens": 8491566 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 27.477, "latency": 1096.935, "stderr": 3.003, "cost_per_test": 3.389451, "token_totals": { "input_tokens": 808874983, "output_tokens": 22652618, "reasoning_tokens": 14192096, "cache_read_tokens": 772656344, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 25.225, "latency": 1052.564, "stderr": 2.921, "cost_per_test": 1.841835, "token_totals": { "input_tokens": 343810087, "output_tokens": 17892803, "reasoning_tokens": 9064037, "cache_read_tokens": 258912640, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 22.523, "latency": 1215.329, "stderr": 2.81, "cost_per_test": 1.293505, "token_totals": { "input_tokens": 139501369, "output_tokens": 13419785, "reasoning_tokens": 10817707, "cache_read_tokens": 123201009, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 21.622, "latency": 2107.543, "stderr": 2.769, "cost_per_test": 0.057497, "token_totals": { "input_tokens": 354503482, "output_tokens": 25688926, "reasoning_tokens": 20934111, "cache_read_tokens": 321129792, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 21.171, "latency": 484.327, "stderr": 2.748, "cost_per_test": 0.164089, "token_totals": { "input_tokens": 619131425, "output_tokens": 26079934, "reasoning_tokens": 20201286, "cache_read_tokens": 614311296, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 20.721, "latency": 360.456, "stderr": 2.726, "cost_per_test": 1.065147, "token_totals": { "input_tokens": 262123547, "output_tokens": 8178370, "reasoning_tokens": 3546375, "cache_read_tokens": 214402702, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 19.369, "latency": 361.208, "stderr": 2.658, "cost_per_test": 0.025854, "token_totals": { "input_tokens": 66323704, "output_tokens": 8047144, "reasoning_tokens": 7044372, "cache_read_tokens": 57123096, "cache_write_tokens": 8985227 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 18.468, "latency": 1760.418, "stderr": 2.61, "cost_per_test": 0.597179, "token_totals": { "input_tokens": 455746580, "output_tokens": 27980610, "reasoning_tokens": 0, "cache_read_tokens": 419705856, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 14.414, "latency": 493.052, "stderr": 2.363, "cost_per_test": 0.109246, "token_totals": { "input_tokens": 447050993, "output_tokens": 9484085, "reasoning_tokens": 5857002, "cache_read_tokens": 431065411, "cache_write_tokens": 15750179 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "terminal_bench_science": { "openai/gpt-6-astra": { "accuracy": 62.857, "latency": 6534.868, "stderr": 5.817, "cost_per_test": 20.802802, "token_totals": { "input_tokens": 810028865, "output_tokens": 19639528, "reasoning_tokens": 8544230, "cache_read_tokens": 386366671, "cache_write_tokens": 18635815 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 52.857, "latency": 5003.987, "stderr": 6.009, "cost_per_test": 3.436948, "token_totals": { "input_tokens": 675620471, "output_tokens": 20718878, "reasoning_tokens": 9185469, "cache_read_tokens": 317832491, "cache_write_tokens": 19972804 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 47.143, "latency": 8856.116, "stderr": 6.009, "cost_per_test": 19.121966, "token_totals": { "input_tokens": 3658605476, "output_tokens": 29236526, "reasoning_tokens": 11843007, "cache_read_tokens": 1700954981, "cache_write_tokens": 128339444 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 47.143, "latency": 9980.148, "stderr": 6.009, "cost_per_test": 32.516187, "token_totals": { "input_tokens": 18222221958, "output_tokens": 58606026, "reasoning_tokens": 24263221, "cache_read_tokens": 9063088192, "cache_write_tokens": 48002507 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 44.286, "latency": 6369.095, "stderr": 5.98, "cost_per_test": 44.904641, "token_totals": { "input_tokens": 3866192418, "output_tokens": 32152416, "reasoning_tokens": 10397231, "cache_read_tokens": 1635454933, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 37.143, "latency": 12919.209, "stderr": 5.817, "cost_per_test": 22.814086, "token_totals": { "input_tokens": 3713083934, "output_tokens": 17062678, "reasoning_tokens": 11556867, "cache_read_tokens": 3578053496, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 30.0, "latency": 4930.428, "stderr": 5.517, "cost_per_test": 5.821477, "token_totals": { "input_tokens": 2212998922, "output_tokens": 14625198, "reasoning_tokens": 5482665, "cache_read_tokens": 1090637986, "cache_write_tokens": 15843344 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 27.143, "latency": 7908.453, "stderr": 5.354, "cost_per_test": 32.543386, "token_totals": { "input_tokens": 6502625236, "output_tokens": 30457719, "reasoning_tokens": 11809096, "cache_read_tokens": 3218944868, "cache_write_tokens": 32354267 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 22.857, "latency": 5584.198, "stderr": 5.055, "cost_per_test": 19.609644, "token_totals": { "input_tokens": 4434041175, "output_tokens": 17847376, "reasoning_tokens": 12044271, "cache_read_tokens": 4400115036, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 20.0, "latency": 6216.027, "stderr": 4.815, "cost_per_test": 7.322071, "token_totals": { "input_tokens": 1324814946, "output_tokens": 8813869, "reasoning_tokens": 2789570, "cache_read_tokens": 648354535, "cache_write_tokens": 14038242 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 11.429, "latency": 4696.051, "stderr": 3.83, "cost_per_test": 4.815933, "token_totals": { "input_tokens": 1016800111, "output_tokens": 10643089, "reasoning_tokens": 3485074, "cache_read_tokens": 496980864, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 10.0, "latency": 5553.191, "stderr": 3.612, "cost_per_test": 17.040638, "token_totals": { "input_tokens": 2242163779, "output_tokens": 18843756, "reasoning_tokens": 6886200, "cache_read_tokens": 1084056704, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 10.0, "latency": 5637.472, "stderr": 3.612, "cost_per_test": 4.523698, "token_totals": { "input_tokens": 9929451142, "output_tokens": 45171153, "reasoning_tokens": 19018280, "cache_read_tokens": 4931170604, "cache_write_tokens": 33540337 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 8.571, "latency": 3311.435, "stderr": 3.37, "cost_per_test": 5.643008, "token_totals": { "input_tokens": 2624221996, "output_tokens": 17012440, "reasoning_tokens": 5862209, "cache_read_tokens": 1271911976, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 5.714, "latency": 12196.691, "stderr": 2.794, "cost_per_test": 30.760832, "token_totals": { "input_tokens": 11082295199, "output_tokens": 33579105, "reasoning_tokens": 13579586, "cache_read_tokens": 5507652660, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 4.286, "latency": 4559.47, "stderr": 2.438, "cost_per_test": 0.240966, "token_totals": { "input_tokens": 1542186407, "output_tokens": 17453747, "reasoning_tokens": 7497116, "cache_read_tokens": 755113085, "cache_write_tokens": 15962745 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 4.286, "latency": 14326.966, "stderr": 2.438, "cost_per_test": 0.244346, "token_totals": { "input_tokens": 4200519429, "output_tokens": 34599179, "reasoning_tokens": 14752104, "cache_read_tokens": 2079818560, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 2.857, "latency": 8997.76, "stderr": 2.006, "cost_per_test": 4.250933, "token_totals": { "input_tokens": 8167134278, "output_tokens": 21535694, "reasoning_tokens": 0, "cache_read_tokens": 4065495040, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 1.429, "latency": 2638.065, "stderr": 1.429, "cost_per_test": 5.907095, "token_totals": { "input_tokens": 1655712288, "output_tokens": 6012954, "reasoning_tokens": 2054169, "cache_read_tokens": 796983036, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 7396.452, "stderr": 0.0, "cost_per_test": 0.581456, "token_totals": { "input_tokens": 2259393866, "output_tokens": 11187767, "reasoning_tokens": 3921207, "cache_read_tokens": 1113668160, "cache_write_tokens": 16009244 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 10621.573, "stderr": 0.0, "cost_per_test": 0.473154, "token_totals": { "input_tokens": 3524356137, "output_tokens": 30097994, "reasoning_tokens": 12785452, "cache_read_tokens": 1759193848, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null } } } }