{ "metadata": { "benchmark": "ProofBench v1.1", "slug": "proof_bench", "description": "Automated theorem proving benchmark", "benchmark_id": "proof_bench", "family": "proof_bench", "version": "1.1", "updated": "2026-10-29", "dataset_type": "private", "industry": "math", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "aristotle/aristotle", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "fireworks/ember-1", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k3", "kimi/kimi-k3.1-pro", "logicalintelligence/alephprover", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M3", "mistralai/mistral-large-4", "mistralai/mistral-medium-3.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "reflection/Beam-1.0", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 100, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 54 }, "tasks": { "overall": { "anthropic/claude-sonnet-5-5": { "accuracy": 100.0, "latency": 347.583, "stderr": 0.0, "cost_per_test": 0.558327, "token_totals": { "input_tokens": 38210470, "output_tokens": 3817407, "reasoning_tokens": 2302419, "cache_read_tokens": 33854803, "cache_write_tokens": 4352571 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 7.7, "avg_query_seconds": 44.9, "median_query_seconds": 33.3, "avg_ttft_seconds": 27.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": 351.035, "stderr": 0.0, "cost_per_test": 0.961648, "token_totals": { "input_tokens": 22741291, "output_tokens": 3445356, "reasoning_tokens": 2381878, "cache_read_tokens": 18792754, "cache_write_tokens": 3946449 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 5.2, "avg_query_seconds": 67.2, "median_query_seconds": 36.5, "avg_ttft_seconds": 53.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 100.0, "latency": 478.303, "stderr": 0.0, "cost_per_test": 2.70347, "token_totals": { "input_tokens": 25650116, "output_tokens": 4101355, "reasoning_tokens": 2989018, "cache_read_tokens": 20844282, "cache_write_tokens": 4803934 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 4.8, "avg_query_seconds": 100.7, "median_query_seconds": 50.8, "avg_ttft_seconds": 74.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "logicalintelligence/alephprover": { "accuracy": 100.0, "latency": 1108.1, "stderr": 0.0, "cost_per_test": 9.3487, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Logical Intelligence", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 99.0, "latency": 284.697, "stderr": 1.0, "cost_per_test": 0.233057, "token_totals": { "input_tokens": 32842768, "output_tokens": 1038054, "reasoning_tokens": 321606, "cache_read_tokens": 28854579, "cache_write_tokens": 3985465 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 9.1, "avg_query_seconds": 31.3, "median_query_seconds": 25.7, "avg_ttft_seconds": 12.0, "output_time_split": { "reasoning_pct": 11.6, "content_pct": 0.0, "tool_call_pct": 88.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 99.0, "latency": 307.867, "stderr": 1.0, "cost_per_test": 1.71238, "token_totals": { "input_tokens": 35641964, "output_tokens": 997178, "reasoning_tokens": 309366, "cache_read_tokens": 29434576, "cache_write_tokens": null }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 8.7, "avg_query_seconds": 35.3, "median_query_seconds": 30.1, "avg_ttft_seconds": 8.8, "output_time_split": { "reasoning_pct": 11.8, "content_pct": 0.0, "tool_call_pct": 88.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 99.0, "latency": 380.246, "stderr": 1.0, "cost_per_test": 0.782978, "token_totals": { "input_tokens": 37238434, "output_tokens": 1694278, "reasoning_tokens": 702938, "cache_read_tokens": 27510930, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 10.8, "avg_query_seconds": 35.3, "median_query_seconds": 27.7, "avg_ttft_seconds": 6.8, "output_time_split": { "reasoning_pct": 92.4, "content_pct": 0.0, "tool_call_pct": 7.6 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 99.0, "latency": 590.886, "stderr": 1.0, "cost_per_test": 1.786125, "token_totals": { "input_tokens": 51018430, "output_tokens": 4769719, "reasoning_tokens": 3301232, "cache_read_tokens": 45129298, "cache_write_tokens": 5887376 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 8.8, "avg_query_seconds": 67.3, "median_query_seconds": 42.9, "avg_ttft_seconds": 46.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 95.0, "latency": 822.739, "stderr": 2.19, "cost_per_test": 5.229282, "token_totals": { "input_tokens": 35941254, "output_tokens": 7112728, "reasoning_tokens": 5678640, "cache_read_tokens": 24973984, "cache_write_tokens": 10966348 }, "token_tasks_measured": 99, "turn_metrics": { "tasks_measured": 99, "avg_turns": 4.7, "avg_query_seconds": 176.7, "median_query_seconds": 102.4, "avg_ttft_seconds": 150.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 94.0, "latency": 695.558, "stderr": 2.387, "cost_per_test": 0.856208, "token_totals": { "input_tokens": 62718296, "output_tokens": 2708964, "reasoning_tokens": 1349584, "cache_read_tokens": 57745168, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 17.7, "avg_query_seconds": 39.2, "median_query_seconds": 26.8, "avg_ttft_seconds": 21.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 91.0, "latency": 496.158, "stderr": 2.876, "cost_per_test": 0.474341, "token_totals": { "input_tokens": 73516483, "output_tokens": 2238130, "reasoning_tokens": 1202518, "cache_read_tokens": 67766784, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 15.0, "avg_query_seconds": 33.0, "median_query_seconds": 18.3, "avg_ttft_seconds": 1.4, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 89.0, "latency": 418.047, "stderr": 3.145, "cost_per_test": 0.173874, "token_totals": { "input_tokens": 42301619, "output_tokens": 9672099, "reasoning_tokens": 8496068, "cache_read_tokens": 29799415, "cache_write_tokens": 12500492 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 4.3, "avg_query_seconds": 96.5, "median_query_seconds": 50.3, "avg_ttft_seconds": 73.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 87.0, "latency": 2070.511, "stderr": 3.38, "cost_per_test": 1.646802, "token_totals": { "input_tokens": 104584764, "output_tokens": 7319059, "reasoning_tokens": 5780416, "cache_read_tokens": 95874048, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 13.4, "avg_query_seconds": 154.0, "median_query_seconds": 76.0, "avg_ttft_seconds": 14.3, "output_time_split": { "reasoning_pct": 79.9, "content_pct": 0.6, "tool_call_pct": 19.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "aristotle/aristotle": { "accuracy": 86.0, "latency": null, "stderr": 3.47, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Harmonic", "harness": null }, "meta/muse-wonder": { "accuracy": 84.0, "latency": 929.053, "stderr": 3.685, "cost_per_test": 1.313345, "token_totals": { "input_tokens": 83508776, "output_tokens": 3687574, "reasoning_tokens": 2180479, "cache_read_tokens": 72750544, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 21.0, "avg_query_seconds": 44.1, "median_query_seconds": 26.0, "avg_ttft_seconds": 29.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 83.0, "latency": 304.509, "stderr": 3.775, "cost_per_test": 0.476321, "token_totals": { "input_tokens": 99039756, "output_tokens": 1363035, "reasoning_tokens": 593322, "cache_read_tokens": 92867287, "cache_write_tokens": 6166643 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 19.4, "avg_query_seconds": 15.7, "median_query_seconds": 13.1, "avg_ttft_seconds": 5.4, "output_time_split": { "reasoning_pct": 15.3, "content_pct": 0.0, "tool_call_pct": 84.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 83.0, "latency": 520.405, "stderr": 3.775, "cost_per_test": 1.438313, "token_totals": { "input_tokens": 123160663, "output_tokens": 1522264, "reasoning_tokens": 541932, "cache_read_tokens": 115350727, "cache_write_tokens": null }, "token_tasks_measured": 99, "turn_metrics": { "tasks_measured": 99, "avg_turns": 18.0, "avg_query_seconds": 28.9, "median_query_seconds": 20.7, "avg_ttft_seconds": 10.1, "output_time_split": { "reasoning_pct": 22.2, "content_pct": 0.0, "tool_call_pct": 77.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 77.0, "latency": 898.079, "stderr": 4.23, "cost_per_test": 1.373684, "token_totals": { "input_tokens": 158153304, "output_tokens": 7432445, "reasoning_tokens": 5551345, "cache_read_tokens": 145688844, "cache_write_tokens": 12459810 }, "token_tasks_measured": 98, "turn_metrics": { "tasks_measured": 98, "avg_turns": 23.7, "avg_query_seconds": 37.8, "median_query_seconds": 23.8, "avg_ttft_seconds": 27.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 75.0, "latency": 1373.837, "stderr": 4.352, "cost_per_test": 0.335717, "token_totals": { "input_tokens": 134713193, "output_tokens": 7030036, "reasoning_tokens": 4997323, "cache_read_tokens": 121668224, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 19.1, "avg_query_seconds": 71.9, "median_query_seconds": 50.3, "avg_ttft_seconds": 3.4, "output_time_split": { "reasoning_pct": 77.7, "content_pct": 0.1, "tool_call_pct": 22.2 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 74.0, "latency": 777.341, "stderr": 4.408, "cost_per_test": 1.274209, "token_totals": { "input_tokens": 241131910, "output_tokens": 1542468, "reasoning_tokens": 704967, "cache_read_tokens": 222728984, "cache_write_tokens": null }, "token_tasks_measured": 99, "turn_metrics": { "tasks_measured": 99, "avg_turns": 21.4, "avg_query_seconds": 36.3, "median_query_seconds": 21.6, "avg_ttft_seconds": 5.4, "output_time_split": { "reasoning_pct": 29.1, "content_pct": 0.0, "tool_call_pct": 70.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 70.0, "latency": 3693.469, "stderr": 4.606, "cost_per_test": 0.24535, "token_totals": { "input_tokens": 214175878, "output_tokens": 15653193, "reasoning_tokens": 13784976, "cache_read_tokens": 192932736, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 13.5, "avg_query_seconds": 273.6, "median_query_seconds": 152.1, "avg_ttft_seconds": 10.4, "output_time_split": { "reasoning_pct": 92.5, "content_pct": 0.2, "tool_call_pct": 7.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 64.0, "latency": 451.481, "stderr": 4.824, "cost_per_test": 0.041637, "token_totals": { "input_tokens": 126053768, "output_tokens": 2652752, "reasoning_tokens": 1912396, "cache_read_tokens": 117050095, "cache_write_tokens": 8996692 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 23.3, "avg_query_seconds": 19.4, "median_query_seconds": 13.8, "avg_ttft_seconds": 7.4, "output_time_split": { "reasoning_pct": 31.2, "content_pct": 0.0, "tool_call_pct": 68.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 63.0, "latency": 3769.808, "stderr": 4.852, "cost_per_test": 0.123277, "token_totals": { "input_tokens": 398251867, "output_tokens": 24214511, "reasoning_tokens": 20726471, "cache_read_tokens": 365944512, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 17.0, "avg_query_seconds": 222.0, "median_query_seconds": 143.6, "avg_ttft_seconds": 8.4, "output_time_split": { "reasoning_pct": 20.4, "content_pct": 0.0, "tool_call_pct": 79.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/ember-1": { "accuracy": 61.0, "latency": 1269.194, "stderr": 4.902, "cost_per_test": 1.737739, "token_totals": { "input_tokens": 111831127, "output_tokens": 7247825, "reasoning_tokens": 5344776, "cache_read_tokens": 100161789, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 15.3, "avg_query_seconds": 82.9, "median_query_seconds": 37.5, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 73.5, "content_pct": 0.4, "tool_call_pct": 26.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 60.0, "latency": 513.687, "stderr": 4.924, "cost_per_test": 0.123288, "token_totals": { "input_tokens": 203002670, "output_tokens": 1885904, "reasoning_tokens": 1035104, "cache_read_tokens": 184422820, "cache_write_tokens": null }, "token_tasks_measured": 97, "turn_metrics": { "tasks_measured": 97, "avg_turns": 25.1, "avg_query_seconds": 20.5, "median_query_seconds": 15.9, "avg_ttft_seconds": 6.8, "output_time_split": { "reasoning_pct": 30.7, "content_pct": 0.0, "tool_call_pct": 69.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 58.0, "latency": 423.698, "stderr": 4.96, "cost_per_test": 0.558508, "token_totals": { "input_tokens": 58411119, "output_tokens": 1397278, "reasoning_tokens": 567660, "cache_read_tokens": 31292904, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 29.9, "avg_query_seconds": 14.2, "median_query_seconds": 10.4, "avg_ttft_seconds": 2.9, "output_time_split": { "reasoning_pct": 91.1, "content_pct": 0.0, "tool_call_pct": 8.9 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 58.0, "latency": 568.696, "stderr": 4.96, "cost_per_test": 0.479019, "token_totals": { "input_tokens": 110539459, "output_tokens": 4529213, "reasoning_tokens": 3431454, "cache_read_tokens": 99565051, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 30.1, "avg_query_seconds": 18.9, "median_query_seconds": 10.8, "avg_ttft_seconds": 4.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 58.0, "latency": 1867.622, "stderr": 4.96, "cost_per_test": 1.503984, "token_totals": { "input_tokens": 207535531, "output_tokens": 10505110, "reasoning_tokens": 9299092, "cache_read_tokens": 190696704, "cache_write_tokens": null }, "token_tasks_measured": 96, "turn_metrics": { "tasks_measured": 96, "avg_turns": 27.1, "avg_query_seconds": 69.0, "median_query_seconds": 22.2, "avg_ttft_seconds": 7.2, "output_time_split": { "reasoning_pct": 87.6, "content_pct": 0.1, "tool_call_pct": 12.2 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 56.0, "latency": 933.981, "stderr": 4.989, "cost_per_test": 0.156045, "token_totals": { "input_tokens": 168698684, "output_tokens": 7445555, "reasoning_tokens": 5646846, "cache_read_tokens": 160683264, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 27.7, "avg_query_seconds": 33.7, "median_query_seconds": 22.3, "avg_ttft_seconds": 1.4, "output_time_split": { "reasoning_pct": 82.8, "content_pct": 0.0, "tool_call_pct": 17.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 55.0, "latency": 534.06, "stderr": 5.0, "cost_per_test": 0.433712, "token_totals": { "input_tokens": 106926796, "output_tokens": 3606435, "reasoning_tokens": 2473887, "cache_read_tokens": 96013349, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 29.7, "avg_query_seconds": 17.9, "median_query_seconds": 11.7, "avg_ttft_seconds": 4.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 54.0, "latency": 546.595, "stderr": 5.009, "cost_per_test": 0.126437, "token_totals": { "input_tokens": 183256713, "output_tokens": 8190207, "reasoning_tokens": 6720807, "cache_read_tokens": 177420288, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 20.2, "avg_query_seconds": 27.1, "median_query_seconds": 20.1, "avg_ttft_seconds": 1.6, "output_time_split": { "reasoning_pct": 84.7, "content_pct": 0.2, "tool_call_pct": 15.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.6": { "accuracy": 51.0, "latency": 861.898, "stderr": 5.024, "cost_per_test": 0.824475, "token_totals": { "input_tokens": 86079791, "output_tokens": 3827061, "reasoning_tokens": 2540626, "cache_read_tokens": 75116288, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 29.4, "avg_query_seconds": 29.3, "median_query_seconds": 17.0, "avg_ttft_seconds": 1.5, "output_time_split": { "reasoning_pct": 73.7, "content_pct": 24.7, "tool_call_pct": 1.6 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 50.0, "latency": 732.365, "stderr": 5.025, "cost_per_test": 0.262008, "token_totals": { "input_tokens": 133104897, "output_tokens": 2434507, "reasoning_tokens": 1527005, "cache_read_tokens": 124716544, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 29.5, "avg_query_seconds": 24.8, "median_query_seconds": 14.5, "avg_ttft_seconds": 1.6, "output_time_split": { "reasoning_pct": 73.5, "content_pct": 0.0, "tool_call_pct": 26.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 49.0, "latency": 2542.672, "stderr": 5.024, "cost_per_test": 2.081538, "token_totals": { "input_tokens": 275938147, "output_tokens": 19360716, "reasoning_tokens": null, "cache_read_tokens": 232831819, "cache_write_tokens": 0 }, "token_tasks_measured": 99, "turn_metrics": { "tasks_measured": 99, "avg_turns": 14.7, "avg_query_seconds": 172.5, "median_query_seconds": 117.7, "avg_ttft_seconds": 4.6, "output_time_split": { "reasoning_pct": 89.9, "content_pct": 0.3, "tool_call_pct": 9.8 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 48.0, "latency": 368.648, "stderr": 5.021, "cost_per_test": 0.602872, "token_totals": { "input_tokens": 69594721, "output_tokens": 1221773, "reasoning_tokens": 738421, "cache_read_tokens": 40351064, "cache_write_tokens": 0 }, "token_tasks_measured": 98, "turn_metrics": { "tasks_measured": 98, "avg_turns": 32.5, "avg_query_seconds": 11.3, "median_query_seconds": 9.0, "avg_ttft_seconds": 1.6, "output_time_split": { "reasoning_pct": 95.5, "content_pct": 0.0, "tool_call_pct": 4.5 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 43.0, "latency": 469.146, "stderr": 4.976, "cost_per_test": 0.442007, "token_totals": { "input_tokens": 104594507, "output_tokens": 3716809, "reasoning_tokens": 2515620, "cache_read_tokens": 94642678, "cache_write_tokens": 0 }, "token_tasks_measured": 96, "turn_metrics": { "tasks_measured": 96, "avg_turns": 32.5, "avg_query_seconds": 14.4, "median_query_seconds": 10.3, "avg_ttft_seconds": 7.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "stepfun/step-5-preview": { "accuracy": 42.0, "latency": 2313.592, "stderr": 4.96, "cost_per_test": 0.714564, "token_totals": { "input_tokens": 258573571, "output_tokens": 13318410, "reasoning_tokens": 0, "cache_read_tokens": 234817792, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 31.1, "avg_query_seconds": 74.3, "median_query_seconds": 34.6, "avg_ttft_seconds": 4.0, "output_time_split": { "reasoning_pct": 89.1, "content_pct": 1.0, "tool_call_pct": 9.8 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 31.0, "latency": 394.476, "stderr": 4.648, "cost_per_test": 0.569863, "token_totals": { "input_tokens": 66676489, "output_tokens": 1338730, "reasoning_tokens": 805865, "cache_read_tokens": 42908380, "cache_write_tokens": 0 }, "token_tasks_measured": 95, "turn_metrics": { "tasks_measured": 95, "avg_turns": 34.4, "avg_query_seconds": 11.5, "median_query_seconds": 9.5, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 89.1, "content_pct": 9.2, "tool_call_pct": 1.7 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 31.0, "latency": 668.86, "stderr": 4.648, "cost_per_test": 0.573991, "token_totals": { "input_tokens": 74344565, "output_tokens": 2301329, "reasoning_tokens": 606173, "cache_read_tokens": 64634880, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 29.1, "avg_query_seconds": 23.0, "median_query_seconds": 16.6, "avg_ttft_seconds": 1.5, "output_time_split": { "reasoning_pct": 41.1, "content_pct": 55.3, "tool_call_pct": 3.6 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 26.0, "latency": 767.177, "stderr": 4.408, "cost_per_test": 0.787951, "token_totals": { "input_tokens": 70796678, "output_tokens": 1422566, "reasoning_tokens": 829928, "cache_read_tokens": 50374400, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 35.9, "avg_query_seconds": 21.3, "median_query_seconds": 12.5, "avg_ttft_seconds": 2.9, "output_time_split": { "reasoning_pct": 94.7, "content_pct": 0.7, "tool_call_pct": 4.7 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 26.0, "latency": 889.059, "stderr": 4.408, "cost_per_test": 0.784253, "token_totals": { "input_tokens": 68260388, "output_tokens": 2100047, "reasoning_tokens": 691664, "cache_read_tokens": 47066256, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 29.9, "avg_query_seconds": 29.7, "median_query_seconds": 19.8, "avg_ttft_seconds": 11.9, "output_time_split": { "reasoning_pct": 56.8, "content_pct": 4.4, "tool_call_pct": 38.8 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 22.0, "latency": 1177.352, "stderr": 4.163, "cost_per_test": 0.089853, "token_totals": { "input_tokens": 109400066, "output_tokens": 5301519, "reasoning_tokens": 0, "cache_read_tokens": 100176384, "cache_write_tokens": null }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 34.2, "avg_query_seconds": 34.4, "median_query_seconds": 17.2, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 82.6, "content_pct": 0.4, "tool_call_pct": 17.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 21.0, "latency": 4274.803, "stderr": 4.094, "cost_per_test": 0.224234, "token_totals": { "input_tokens": 703145622, "output_tokens": 33321229, "reasoning_tokens": null, "cache_read_tokens": 644046720, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 23.2, "avg_query_seconds": 184.2, "median_query_seconds": 74.2, "avg_ttft_seconds": 3.2, "output_time_split": { "reasoning_pct": 92.4, "content_pct": 1.6, "tool_call_pct": 5.9 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 18.0, "latency": 266.658, "stderr": 3.861, "cost_per_test": null, "token_totals": { "input_tokens": 64179687, "output_tokens": 1784980, "reasoning_tokens": 1415418, "cache_read_tokens": 56986688, "cache_write_tokens": null }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 36.7, "avg_query_seconds": 7.2, "median_query_seconds": 3.5, "avg_ttft_seconds": 1.6, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 18.0, "latency": 718.66, "stderr": 3.861, "cost_per_test": 0.419224, "token_totals": { "input_tokens": 162739395, "output_tokens": 4876274, "reasoning_tokens": 1498488, "cache_read_tokens": 151053885, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 33.8, "avg_query_seconds": 21.2, "median_query_seconds": 11.0, "avg_ttft_seconds": 2.0, "output_time_split": { "reasoning_pct": 87.8, "content_pct": 1.6, "tool_call_pct": 10.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 16.0, "latency": 423.45, "stderr": 3.685, "cost_per_test": 0.142045, "token_totals": { "input_tokens": 61091499, "output_tokens": 1611579, "reasoning_tokens": 1200572, "cache_read_tokens": 57067520, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 37.5, "avg_query_seconds": 11.3, "median_query_seconds": 7.2, "avg_ttft_seconds": 1.4, "output_time_split": { "reasoning_pct": 85.7, "content_pct": 0.0, "tool_call_pct": 14.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 16.0, "latency": 1255.813, "stderr": 3.685, "cost_per_test": 0.05707, "token_totals": { "input_tokens": 139624413, "output_tokens": 11716945, "reasoning_tokens": 10908282, "cache_read_tokens": 125621632, "cache_write_tokens": null }, "token_tasks_measured": 98, "turn_metrics": { "tasks_measured": 98, "avg_turns": 33.1, "avg_query_seconds": 38.0, "median_query_seconds": 14.8, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 92.1, "content_pct": 0.3, "tool_call_pct": 7.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 16.0, "latency": 1453.872, "stderr": 3.685, "cost_per_test": 2.298832, "token_totals": { "input_tokens": 358351531, "output_tokens": 16902495, "reasoning_tokens": 14195905, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 30.0, "avg_query_seconds": 48.5, "median_query_seconds": 24.2, "avg_ttft_seconds": 4.8, "output_time_split": { "reasoning_pct": 84.2, "content_pct": 0.4, "tool_call_pct": 15.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 10.0, "latency": 1258.806, "stderr": 3.015, "cost_per_test": 0.329279, "token_totals": { "input_tokens": 59543255, "output_tokens": 1823695, "reasoning_tokens": null, "cache_read_tokens": 19971584, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 36.9, "avg_query_seconds": 34.1, "median_query_seconds": 21.1, "avg_ttft_seconds": 5.1, "output_time_split": { "reasoning_pct": 99.8, "content_pct": 0.0, "tool_call_pct": 0.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 9.0, "latency": 479.048, "stderr": 2.876, "cost_per_test": 1.302032, "token_totals": { "input_tokens": 66324649, "output_tokens": 1838640, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 87, "turn_metrics": { "tasks_measured": 87, "avg_turns": 36.5, "avg_query_seconds": 13.1, "median_query_seconds": 7.8, "avg_ttft_seconds": 4.5, "output_time_split": { "reasoning_pct": 96.2, "content_pct": 3.2, "tool_call_pct": 0.5 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 6.0, "latency": 507.301, "stderr": 2.387, "cost_per_test": 0.082234, "token_totals": { "input_tokens": 62832266, "output_tokens": 2596974, "reasoning_tokens": null, "cache_read_tokens": 57261056, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 37.7, "avg_query_seconds": 13.4, "median_query_seconds": 8.7, "avg_ttft_seconds": 1.8, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.1, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-2.5": { "accuracy": 3.0, "latency": 201.725, "stderr": 1.714, "cost_per_test": 0.044105, "token_totals": { "input_tokens": 54269237, "output_tokens": 1560820, "reasoning_tokens": 1435858, "cache_read_tokens": 42299649, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 33.3, "avg_query_seconds": 6.0, "median_query_seconds": 5.0, "avg_ttft_seconds": 1.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 436.43, "stderr": 0.0, "cost_per_test": 0.198639, "token_totals": { "input_tokens": 56824656, "output_tokens": 1417408, "reasoning_tokens": null, "cache_read_tokens": 51447296, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 39.4, "avg_query_seconds": 11.1, "median_query_seconds": 7.5, "avg_ttft_seconds": 1.6, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 919.248, "stderr": 0.0, "cost_per_test": 0.015926, "token_totals": { "input_tokens": 43944505, "output_tokens": 5131368, "reasoning_tokens": null, "cache_read_tokens": 40772695, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 36.2, "avg_query_seconds": 25.4, "median_query_seconds": 9.4, "avg_ttft_seconds": 4.0, "output_time_split": { "reasoning_pct": 72.4, "content_pct": 22.2, "tool_call_pct": 5.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }