{ "metadata": { "benchmark": "ProofBench v1", "slug": "proof_bench_v1", "description": "Automated theorem proving benchmark", "benchmark_id": "proof_bench_v1", "family": "proof_bench", "version": "1", "updated": "2026-08-05", "dataset_type": "private", "industry": "math", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3.6-27b", "alibaba/qwen3.7-max", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "anthropic/claude-fable-5", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "aristotle/aristotle", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "grok/grok-4-1-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-codex-max", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "poolside/laguna-m.1", "poolside/laguna-xs.2", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "zai/glm-4.7", "zai/glm-5.1", "zai/glm-5.2" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": true, "partner": false, "total_models": 61 }, "tasks": { "overall": { "anthropic/claude-opus-5": { "accuracy": 78.0, "latency": 722.717, "stderr": 4.163, "cost_per_test": 2.10619, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 77.0, "latency": 516.531, "stderr": 4.208, "cost_per_test": 1.736601, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 77.0, "latency": 918.917, "stderr": 4.23, "cost_per_test": 5.451518, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 71.0, "latency": 421.678, "stderr": 4.56, "cost_per_test": 0.940019, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "aristotle/aristotle": { "accuracy": 71.0, "latency": 5820.0, "stderr": 4.5, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Harmonic", "harness": null }, "kimi/kimi-k3": { "accuracy": 70.0, "latency": 2013.12, "stderr": 4.606, "cost_per_test": 1.666846, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 69.0, "latency": 1166.322, "stderr": 4.648, "cost_per_test": 2.293129, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 66.0, "latency": 1014.587, "stderr": 4.761, "cost_per_test": 2.248759, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 57.0, "latency": 656.965, "stderr": 4.976, "cost_per_test": 0.538921, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 56.0, "latency": 2751.526, "stderr": 4.964, "cost_per_test": 3.202015, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 54.0, "latency": 387.84, "stderr": 5.009, "cost_per_test": 0.089865, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 54.0, "latency": 679.708, "stderr": 4.984, "cost_per_test": 1.807786, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 50.0, "latency": 2953.443, "stderr": 5.0, "cost_per_test": 1.68179, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 50.0, "latency": 3973.892, "stderr": 5.0, "cost_per_test": 2.703583, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 49.0, "latency": 717.111, "stderr": 5.024, "cost_per_test": 0.067312, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 45.0, "latency": 5463.593, "stderr": 4.975, "cost_per_test": 2.162108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 39.0, "latency": 600.628, "stderr": 4.902, "cost_per_test": 0.027176, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 39.0, "latency": 1056.902, "stderr": 4.902, "cost_per_test": 0.519718, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 36.0, "latency": 346.937, "stderr": 4.824, "cost_per_test": 0.522059, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 36.0, "latency": 3757.671, "stderr": 4.8, "cost_per_test": 1.267043, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 35.0, "latency": 0.0, "stderr": 4.794, "cost_per_test": 0.0, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 30.0, "latency": 460.33, "stderr": 4.606, "cost_per_test": 0.484548, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 29.0, "latency": 413.679, "stderr": 4.56, "cost_per_test": 1.399975, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 26.0, "latency": 771.408, "stderr": 4.408, "cost_per_test": 2.462069, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 26.0, "latency": 2166.981, "stderr": 4.386, "cost_per_test": 0.525953, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 24.0, "latency": 1267.15, "stderr": 4.292, "cost_per_test": 0.071554, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 22.222, "latency": 4755.746, "stderr": 4.178, "cost_per_test": 0.584577, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 21.0, "latency": 1410.671, "stderr": 4.073, "cost_per_test": 0.249441, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 20.0, "latency": 10383.833, "stderr": 4.0, "cost_per_test": 1.229076, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 19.0, "latency": 1394.151, "stderr": 3.943, "cost_per_test": 0.371186, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 19.0, "latency": 12300.813, "stderr": 3.923, "cost_per_test": 0.65341, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 18.0, "latency": 4579.325, "stderr": 3.842, "cost_per_test": 0.852361, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 17.0, "latency": 478.958, "stderr": 3.756, "cost_per_test": 0.067086, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 16.0, "latency": 1386.754, "stderr": 3.666, "cost_per_test": 0.295811, "temperature": null, "top_p": 0.95, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 16.0, "latency": 1943.239, "stderr": 3.685, "cost_per_test": 0.074885, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 15.0, "latency": 1141.548, "stderr": 3.571, "cost_per_test": 0.327286, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 15.0, "latency": 1493.817, "stderr": 3.571, "cost_per_test": 1.059689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 14.0, "latency": 494.946, "stderr": 3.47, "cost_per_test": 0.31815, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 13.0, "latency": 290.614, "stderr": 3.38, "cost_per_test": 0.132788, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 12.0, "latency": 3729.837, "stderr": 3.25, "cost_per_test": 0.049218, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 11.0, "latency": 1643.019, "stderr": 3.129, "cost_per_test": 0.244122, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 10.0, "latency": 402.484, "stderr": 3.015, "cost_per_test": 1.501427, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 10.0, "latency": 518.756, "stderr": 3.0, "cost_per_test": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 9.0, "latency": 3184.213, "stderr": 2.862, "cost_per_test": 0.527356, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 9.0, "latency": 3349.817, "stderr": 2.862, "cost_per_test": 0.22021, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 8.0, "latency": 695.339, "stderr": 2.713, "cost_per_test": 0.512041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 8.0, "latency": 3981.084, "stderr": 2.713, "cost_per_test": 0.11773, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 6.0, "latency": 503.701, "stderr": 2.387, "cost_per_test": 0.040744, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "zai/glm-4.7": { "accuracy": 6.0, "latency": 3364.789, "stderr": 2.375, "cost_per_test": 0.14404, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 5.0, "latency": 525.432, "stderr": 2.179, "cost_per_test": 0.015894, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 5.0, "latency": 917.84, "stderr": 2.19, "cost_per_test": 0.075386, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 4.0, "latency": 1421.041, "stderr": 1.96, "cost_per_test": 0.096578, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 4.0, "latency": 3441.448, "stderr": 1.96, "cost_per_test": 0.128469, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 4.0, "latency": 4514.143, "stderr": 1.96, "cost_per_test": 0.057497, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 3.0, "latency": 1634.068, "stderr": 1.706, "cost_per_test": 0.04293, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 2.0, "latency": 431.228, "stderr": 1.407, "cost_per_test": 0.201396, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 2.0, "latency": 2552.288, "stderr": 1.407, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 315.721, "stderr": 0.0, "cost_per_test": 0.024998, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 423.12, "stderr": 0.0, "cost_per_test": 0.067403, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 47.0, "latency": 1998.736, "stderr": 5.016, "cost_per_test": 1.576343, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 45.0, "latency": 908.033, "stderr": 5.0, "cost_per_test": 0.759043, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } } } }