{ "metadata": { "benchmark": "MysteryMechanism", "slug": "mysterymechanism", "description": "Can agents rediscover sealed mathematical mechanisms through bounded experiments?", "benchmark_id": "mysterymechanism", "family": "mysterymechanism", "version": "1", "updated": "2026-10-29", "dataset_type": "private", "industry": "science", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3.8-max", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4.1-flash", "fireworks/ember-1", "google/gemini-3.1-pro-preview", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.6", "grok/grok-4.7", "kimi/kimi-k3.1-pro", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_3_max", "mistralai/mistral-large-4", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "reflection/Beam-1.0", "stepfun/step-5-preview", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 222, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 29 }, "tasks": { "overall": { "openai/gpt-6-astra": { "accuracy": 53.153, "latency": 488.597, "stderr": 3.357, "cost_per_test": 1.559704, "token_totals": { "input_tokens": 34970642, "output_tokens": 4706731, "reasoning_tokens": 3583655, "cache_read_tokens": 28320058, "cache_write_tokens": 6436778 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 12.2, "avg_query_seconds": 39.9, "median_query_seconds": 24.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 49.55, "latency": 1178.454, "stderr": 3.363, "cost_per_test": 4.619754, "token_totals": { "input_tokens": 658569339, "output_tokens": 29182250, "reasoning_tokens": 23537093, "cache_read_tokens": 619837485, "cache_write_tokens": 38715822 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 35.1, "avg_query_seconds": 33.6, "median_query_seconds": 20.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 49.099, "latency": 1410.236, "stderr": 3.363, "cost_per_test": 3.45323, "token_totals": { "input_tokens": 1091907970, "output_tokens": 42938841, "reasoning_tokens": 35532465, "cache_read_tokens": 1040231244, "cache_write_tokens": 51657746 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 41.7, "avg_query_seconds": 33.8, "median_query_seconds": 21.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 47.748, "latency": 1060.876, "stderr": 3.36, "cost_per_test": 5.631518, "token_totals": { "input_tokens": 217155354, "output_tokens": 17938847, "reasoning_tokens": 13904415, "cache_read_tokens": 192747948, "cache_write_tokens": 24397474 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 21.2, "avg_query_seconds": 50.0, "median_query_seconds": 27.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 46.396, "latency": 437.025, "stderr": 3.355, "cost_per_test": 0.284881, "token_totals": { "input_tokens": 32268619, "output_tokens": 4481195, "reasoning_tokens": 3364816, "cache_read_tokens": 25888801, "cache_write_tokens": 6166336 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 11.8, "avg_query_seconds": 37.2, "median_query_seconds": 24.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 45.495, "latency": 1494.947, "stderr": 3.35, "cost_per_test": 4.594426, "token_totals": { "input_tokens": 901582710, "output_tokens": 21508801, "reasoning_tokens": 10866656, "cache_read_tokens": 793827453, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 52.7, "avg_query_seconds": 28.4, "median_query_seconds": 25.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 37.387, "latency": 1012.543, "stderr": 3.255, "cost_per_test": 3.280693, "token_totals": { "input_tokens": 260014806, "output_tokens": 18050903, "reasoning_tokens": 13538433, "cache_read_tokens": 234441635, "cache_write_tokens": 25563677 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 21.4, "avg_query_seconds": 47.4, "median_query_seconds": 29.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 36.486, "latency": 322.237, "stderr": 3.238, "cost_per_test": 1.731041, "token_totals": { "input_tokens": 827206914, "output_tokens": 17858080, "reasoning_tokens": 9860899, "cache_read_tokens": 733670345, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 70.6, "avg_query_seconds": 4.6, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 36.036, "latency": 451.803, "stderr": 3.23, "cost_per_test": 0.851846, "token_totals": { "input_tokens": 483835714, "output_tokens": 20212161, "reasoning_tokens": 13403759, "cache_read_tokens": 455987667, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 37.0, "avg_query_seconds": 12.2, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 35.135, "latency": 805.031, "stderr": 3.211, "cost_per_test": 0.541024, "token_totals": { "input_tokens": 910086657, "output_tokens": 41490334, "reasoning_tokens": 33895469, "cache_read_tokens": 861339525, "cache_write_tokens": 48729800 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 38.0, "avg_query_seconds": 21.2, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 35.135, "latency": 2494.329, "stderr": 3.211, "cost_per_test": 4.163398, "token_totals": { "input_tokens": 707253940, "output_tokens": 22089086, "reasoning_tokens": 14663498, "cache_read_tokens": 617506072, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 39.5, "avg_query_seconds": 63.2, "median_query_seconds": 41.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 33.333, "latency": 522.744, "stderr": 3.171, "cost_per_test": 0.96335, "token_totals": { "input_tokens": 130077211, "output_tokens": 6106439, "reasoning_tokens": 3933893, "cache_read_tokens": 121397470, "cache_write_tokens": 8456854 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 25.9, "avg_query_seconds": 20.2, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 30.631, "latency": 1674.514, "stderr": 3.101, "cost_per_test": 1.309658, "token_totals": { "input_tokens": 161102384, "output_tokens": 23854775, "reasoning_tokens": 17588077, "cache_read_tokens": 116392832, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 26.4, "avg_query_seconds": 63.4, "median_query_seconds": 42.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 30.18, "latency": 373.426, "stderr": 3.088, "cost_per_test": 0.460931, "token_totals": { "input_tokens": 73446321, "output_tokens": 6771421, "reasoning_tokens": 5398618, "cache_read_tokens": 64736689, "cache_write_tokens": 8491566 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 18.6, "avg_query_seconds": 20.0, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 27.477, "latency": 1096.935, "stderr": 3.003, "cost_per_test": 3.389451, "token_totals": { "input_tokens": 808874983, "output_tokens": 22652618, "reasoning_tokens": 14192096, "cache_read_tokens": 772656344, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 45.4, "avg_query_seconds": 24.2, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 26.126, "latency": 359.9, "stderr": 2.955, "cost_per_test": 0.62244, "token_totals": { "input_tokens": 154914034, "output_tokens": 8456997, "reasoning_tokens": 6568961, "cache_read_tokens": 142342400, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 21.7, "avg_query_seconds": 16.6, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 25.225, "latency": 1052.564, "stderr": 2.921, "cost_per_test": 1.841835, "token_totals": { "input_tokens": 343810087, "output_tokens": 17892803, "reasoning_tokens": 9064037, "cache_read_tokens": 258912640, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 28.5, "avg_query_seconds": 37.0, "median_query_seconds": 32.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 23.874, "latency": 2268.77, "stderr": 2.868, "cost_per_test": 1.309463, "token_totals": { "input_tokens": 361931252, "output_tokens": 24136453, "reasoning_tokens": 19064213, "cache_read_tokens": 330274560, "cache_write_tokens": null }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 25.9, "avg_query_seconds": 87.7, "median_query_seconds": 53.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 22.973, "latency": 1841.98, "stderr": 2.83, "cost_per_test": 1.531337, "token_totals": { "input_tokens": 527457047, "output_tokens": 34798138, "reasoning_tokens": 29570752, "cache_read_tokens": 483855085, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 27.4, "avg_query_seconds": 67.3, "median_query_seconds": 29.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/ember-1": { "accuracy": 22.523, "latency": 1215.329, "stderr": 2.81, "cost_per_test": 1.293505, "token_totals": { "input_tokens": 139501369, "output_tokens": 13419785, "reasoning_tokens": 10817707, "cache_read_tokens": 123201009, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 18.0, "avg_query_seconds": 67.3, "median_query_seconds": 33.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 22.072, "latency": 144.311, "stderr": 2.79, "cost_per_test": null, "token_totals": { "input_tokens": 211587470, "output_tokens": 8130482, "reasoning_tokens": 4665177, "cache_read_tokens": 194116800, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 40.8, "avg_query_seconds": 3.5, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 21.622, "latency": 2107.543, "stderr": 2.769, "cost_per_test": 0.057497, "token_totals": { "input_tokens": 354503482, "output_tokens": 25688926, "reasoning_tokens": 20934111, "cache_read_tokens": 321129792, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 25.2, "avg_query_seconds": 83.7, "median_query_seconds": 38.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 21.171, "latency": 484.327, "stderr": 2.748, "cost_per_test": 0.164089, "turn_metrics": { "tasks_measured": 221, "avg_turns": 36.9, "avg_query_seconds": 13.1, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 20.721, "latency": 360.456, "stderr": 2.726, "cost_per_test": 1.065147, "token_totals": { "input_tokens": 262123547, "output_tokens": 8178370, "reasoning_tokens": 3546375, "cache_read_tokens": 214402702, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 46.0, "avg_query_seconds": 7.8, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 19.369, "latency": 361.208, "stderr": 2.658, "cost_per_test": 0.025854, "token_totals": { "input_tokens": 66323704, "output_tokens": 8047144, "reasoning_tokens": 7044372, "cache_read_tokens": 57123096, "cache_write_tokens": 8985227 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 14.6, "avg_query_seconds": 24.7, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 18.468, "latency": 1760.418, "stderr": 2.61, "cost_per_test": 0.597179, "token_totals": { "input_tokens": 455746580, "output_tokens": 27980610, "reasoning_tokens": 0, "cache_read_tokens": 419705856, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 27.6, "avg_query_seconds": 63.7, "median_query_seconds": 37.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 15.315, "latency": 2639.688, "stderr": 2.423, "cost_per_test": 0.153721, "token_totals": { "input_tokens": 215604422, "output_tokens": 24759918, "reasoning_tokens": 20975198, "cache_read_tokens": 188231296, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 19.0, "avg_query_seconds": 139.2, "median_query_seconds": 66.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 14.414, "latency": 493.052, "stderr": 2.363, "cost_per_test": 0.109246, "token_totals": { "input_tokens": 447050993, "output_tokens": 9484085, "reasoning_tokens": 5857002, "cache_read_tokens": 431065411, "cache_write_tokens": 15750179 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 44.7, "avg_query_seconds": 11.0, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 12.162, "latency": 2136.737, "stderr": 2.199, "cost_per_test": 1.06223, "token_totals": { "input_tokens": 817513750, "output_tokens": 52587714, "reasoning_tokens": 0, "cache_read_tokens": 709331456, "cache_write_tokens": 0 }, "token_tasks_measured": 222, "turn_metrics": { "tasks_measured": 222, "avg_turns": 30.0, "avg_query_seconds": 71.3, "median_query_seconds": 37.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } } } }