{ "metadata": { "benchmark": "MATH 500", "slug": "math500", "description": "Academic math benchmark on probability, algebra, and trigonometry", "benchmark_id": "math500", "family": "math500", "version": "1", "updated": "2026-01-09", "dataset_type": "public", "industry": "math", "tasks": { "overall": "Overall" }, "models": [ "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "cohere/command-a-03-2025", "fireworks/deepseek-r1", "fireworks/deepseek-v3", "fireworks/deepseek-v3-0324", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/qwen3-235b-a22b", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.0-flash-exp", "google/gemini-2.0-flash-thinking-exp-01-21", "google/gemini-2.5-flash-preview-04-17", "google/gemini-2.5-flash-preview-04-17-thinking", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-pro-preview", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "minimax/MiniMax-M2.1", "mistralai/mistral-large-2411", "mistralai/mistral-medium-2505", "mistralai/mistral-small-2402", "mistralai/mistral-small-2503", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", "together/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", "together/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.5" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": false, "total_models": 60 }, "tasks": { "overall": { "google/gemini-3-pro-preview": { "accuracy": 96.4, "latency": 32.146, "stderr": 0.833, "cost_per_test": 0.050514, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-0709": { "accuracy": 96.2, "latency": 27.261, "stderr": 0.855, "cost_per_test": 0.037425, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 96.0, "latency": 32.217, "stderr": 0.876, "cost_per_test": 0.020071, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 95.4, "latency": 45.979, "stderr": 0.937, "cost_per_test": 0.202262, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 95.2, "latency": 25.828, "stderr": 0.956, "cost_per_test": 0.007941, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 94.8, "latency": 12.057, "stderr": 0.993, "cost_per_test": 0.003686, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "fireworks/gpt-oss-120b": { "accuracy": 94.8, "latency": 16.275, "stderr": 0.993, "cost_per_test": 0.001381, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-2025-04-16": { "accuracy": 94.6, "latency": 16.591, "stderr": 1.011, "cost_per_test": 0.010795, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/qwen3-235b-a22b": { "accuracy": 94.6, "latency": 142.754, "stderr": 1.011, "cost_per_test": 0.003361, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o4-mini-2025-04-16": { "accuracy": 94.2, "latency": 12.536, "stderr": 1.045, "cost_per_test": 0.004784, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 94.2, "latency": 22.766, "stderr": 1.045, "cost_per_test": 0.001879, "temperature": 0.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-20b": { "accuracy": 94.2, "latency": 31.472, "stderr": 1.045, "cost_per_test": 0.000955, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 94.2, "latency": 37.346, "stderr": 1.045, "cost_per_test": 0.002985, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "zai/glm-4.5": { "accuracy": 94.0, "latency": 84.928, "stderr": 1.062, "cost_per_test": 0.012744, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 93.8, "latency": 22.396, "stderr": 1.078, "cost_per_test": 0.001664, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 93.8, "latency": 63.474, "stderr": 1.078, "cost_per_test": 0.068378, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 93.0, "latency": 30.932, "stderr": 1.141, "cost_per_test": 0.045983, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-r1": { "accuracy": 92.2, "latency": 156.474, "stderr": 2.364, "cost_per_test": 0.010886, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-mini-2025-01-31": { "accuracy": 91.8, "latency": 14.365, "stderr": 2.417, "cost_per_test": 0.007661, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 91.8, "latency": 23.657, "stderr": 1.227, "cost_per_test": 0.011241, "temperature": 0.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 91.6, "latency": 9.496, "stderr": 1.24, "cost_per_test": 0.006461, "temperature": 0.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 91.6, "latency": 94.242, "stderr": 1.24, "cost_per_test": 0.125312, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 91.4, "latency": 42.828, "stderr": 1.254, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-opus-4-20250514": { "accuracy": 90.4, "latency": 14.812, "stderr": 1.318, "cost_per_test": 0.045429, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 90.4, "latency": 23.537, "stderr": 2.591, "cost_per_test": 0.095269, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 90.323, "latency": 10.028, "stderr": 1.288, "cost_per_test": 0.010201, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-3": { "accuracy": 89.8, "latency": 9.085, "stderr": 1.354, "cost_per_test": 0.012781, "temperature": 0.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.0-flash-exp": { "accuracy": 89.0, "latency": 3.352, "stderr": 2.748, "cost_per_test": 0.0002, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "minimax/MiniMax-M2.1": { "accuracy": 89.0, "latency": 59.692, "stderr": 1.399, "cost_per_test": 0.004642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "fireworks/deepseek-v3-0324": { "accuracy": 88.6, "latency": 12.545, "stderr": 1.421, "cost_per_test": 0.00074, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.0-flash-001": { "accuracy": 88.0, "latency": 3.367, "stderr": 2.852, "cost_per_test": 0.00026, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 88.0, "latency": 5.76, "stderr": 1.453, "cost_per_test": 0.00122, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 87.2, "latency": 12.53, "stderr": 1.494, "cost_per_test": 0.009426, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-medium-2505": { "accuracy": 87.0, "latency": 14.13, "stderr": 1.504, "cost_per_test": 0.001841, "temperature": 0.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 85.2, "latency": 7.469, "stderr": 1.588, "cost_per_test": 0.00059, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-2.0-flash-thinking-exp-01-21": { "accuracy": 84.6, "latency": 11.801, "stderr": 3.163, "cost_per_test": 0.000387, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-1.5-pro-002": { "accuracy": 82.8, "latency": 5.021, "stderr": 3.305, "cost_per_test": 0.002164, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3": { "accuracy": 80.4, "latency": 7.94, "stderr": 1.775, "cost_per_test": 0.000563, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 80.2, "latency": 3.372, "stderr": 1.782, "cost_per_test": 0.000314, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 79.2, "latency": 10.523, "stderr": 1.815, "cost_per_test": 0.00043, "temperature": 0.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "google/gemini-1.5-flash-002": { "accuracy": 78.8, "latency": 2.645, "stderr": 3.576, "cost_per_test": 0.00012, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-2-1212": { "accuracy": 78.4, "latency": 20.44, "stderr": 3.6, "cost_per_test": 0.006277, "temperature": 0.0, "top_p": 0.95, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 76.8, "latency": 5.527, "stderr": 1.888, "cost_per_test": 0.006868, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "cohere/command-a-03-2025": { "accuracy": 76.2, "latency": 8.665, "stderr": 1.904, "cost_per_test": 0.005983, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "openai/gpt-4o-2024-08-06": { "accuracy": 75.2, "latency": 12.291, "stderr": 3.776, "cost_per_test": 0.005641, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-large-2411": { "accuracy": 74.4, "latency": 9.928, "stderr": 3.815, "cost_per_test": 0.003323, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-2024-11-20": { "accuracy": 74.0, "latency": 12.804, "stderr": 3.834, "cost_per_test": 0.008846, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 73.4, "latency": 5.414, "stderr": 3.862, "cost_per_test": 0.000698, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 72.6, "latency": 6.304, "stderr": 3.898, "cost_per_test": 0.00035, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 72.4, "latency": 4.628, "stderr": 3.907, "cost_per_test": 0.004988, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": { "accuracy": 71.4, "latency": 45.304, "stderr": 3.949, "cost_per_test": 0.002967, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 71.2, "latency": 12.55, "stderr": 2.025, "cost_per_test": null, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/mistral-small-2402": { "accuracy": 70.6, "latency": 4.885, "stderr": 3.981, "cost_per_test": 0.000327, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 70.2, "latency": 9.885, "stderr": 2.046, "cost_per_test": 0.000893, "temperature": 0.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-small-2503": { "accuracy": 68.4, "latency": 6.406, "stderr": 2.079, "cost_per_test": 0.000158, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "together/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { "accuracy": 65.0, "latency": 9.201, "stderr": 4.166, "cost_per_test": 0.001035, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 64.2, "latency": 5.128, "stderr": 4.188, "cost_per_test": 0.001498, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "ai21labs/jamba-large-1.6": { "accuracy": 54.8, "latency": 13.007, "stderr": 2.226, "cost_per_test": 0.00395, "temperature": 0.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "together/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { "accuracy": 44.4, "latency": 5.841, "stderr": 4.339, "cost_per_test": 0.000351, "temperature": 0.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 25.4, "latency": 4.859, "stderr": 1.947, "cost_per_test": 0.000306, "temperature": 0.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } } } }