{ "metadata": { "benchmark": "MGSM", "slug": "mgsm", "description": "A multilingual benchmark for mathematical questions.", "benchmark_id": "mgsm", "family": "mgsm", "version": "1", "updated": "2026-01-09", "dataset_type": "public", "industry": "math", "tasks": { "overall": "Overall", "mgsm_en": "English", "mgsm_ja": "Japanese", "mgsm_zh": "Chinese", "mgsm_bn": "Bengali", "mgsm_ru": "Russian", "mgsm_de": "German", "mgsm_fr": "French", "mgsm_th": "Thai", "mgsm_es": "Spanish", "mgsm_te": "Telugu", "mgsm_sw": "Swahili" }, "models": [ "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-preview", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "cohere/command-a-03-2025", "fireworks/deepseek-r1", "fireworks/deepseek-v3", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/qwen3-235b-a22b", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "kimi/kimi-k2-thinking", "minimax/MiniMax-M2.1", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2411", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-small-2402", "mistralai/mistral-small-2503", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": false, "total_models": 75 }, "tasks": { "overall": { "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 95.2, "latency": 11.56, "stderr": 1.284, "cost_per_test": 0.063151, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 94.764, "latency": 5.559, "stderr": 1.35, "cost_per_test": 0.027118, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 94.436, "latency": 14.281, "stderr": 1.429, "cost_per_test": 0.054705, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 94.327, "latency": 10.272, "stderr": 1.416, "cost_per_test": 0.011286, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 94.218, "latency": 7.897, "stderr": 1.456, "cost_per_test": 0.027189, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 94.0, "latency": 10.596, "stderr": 1.416, "cost_per_test": 0.010576, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-pro-preview": { "accuracy": 93.927, "latency": 12.9, "stderr": 1.413, "cost_per_test": 0.018166, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-20250514": { "accuracy": 93.782, "latency": 10.1, "stderr": 1.499, "cost_per_test": 0.027191, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o4-mini-2025-04-16": { "accuracy": 93.418, "latency": 8.067, "stderr": 1.507, "cost_per_test": 0.002905, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-flash-preview": { "accuracy": 93.309, "latency": 14.02, "stderr": 1.456, "cost_per_test": 0.007116, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 93.018, "latency": 5.987, "stderr": 1.563, "cost_per_test": 0.00576, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 92.982, "latency": 8.347, "stderr": 1.536, "cost_per_test": 0.00606, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 92.982, "latency": 23.512, "stderr": 1.568, "cost_per_test": 0.025944, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 92.836, "latency": 20.353, "stderr": 1.559, "cost_per_test": 0.014278, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 92.582, "latency": 4.1, "stderr": 1.616, "cost_per_test": 0.004012, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 92.582, "latency": 12.573, "stderr": 1.59, "cost_per_test": 0.003698, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "fireworks/qwen3-235b-a22b": { "accuracy": 92.473, "latency": 31.899, "stderr": 1.625, "cost_per_test": 0.001108, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 92.436, "latency": 2.714, "stderr": 1.635, "cost_per_test": 0.000218, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 92.4, "latency": 4.861, "stderr": 1.626, "cost_per_test": 0.00566, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-r1": { "accuracy": 92.254, "latency": 10.167, "stderr": 1.655, "cost_per_test": 0.004123, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 92.146, "latency": 9.624, "stderr": 1.649, "cost_per_test": 0.006874, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3": { "accuracy": 92.146, "latency": 15.681, "stderr": 1.663, "cost_per_test": 0.000364, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3-max-preview": { "accuracy": 92.146, "latency": 21.159, "stderr": 1.644, "cost_per_test": 0.002344, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "fireworks/gpt-oss-120b": { "accuracy": 92.036, "latency": 5.793, "stderr": 1.642, "cost_per_test": 0.000659, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3-max": { "accuracy": 91.818, "latency": 11.451, "stderr": 1.677, "cost_per_test": 0.002856, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/o3-2025-04-16": { "accuracy": 91.746, "latency": 6.76, "stderr": 1.673, "cost_per_test": 0.003754, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3-0324": { "accuracy": 91.673, "latency": 18.032, "stderr": 1.696, "cost_per_test": 0.000395, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-3": { "accuracy": 91.346, "latency": 5.831, "stderr": 1.72, "cost_per_test": 0.00567, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/o3-mini-2025-01-31": { "accuracy": 91.346, "latency": 15.976, "stderr": 1.718, "cost_per_test": 0.008194, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 91.091, "latency": 3.008, "stderr": 1.765, "cost_per_test": 0.000515, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 90.946, "latency": 11.83, "stderr": 1.766, "cost_per_test": 0.001001, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-4-0709": { "accuracy": 90.909, "latency": 116.619, "stderr": 1.778, "cost_per_test": 0.026131, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 90.873, "latency": 3.853, "stderr": 1.781, "cost_per_test": 0.000639, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-medium-2505": { "accuracy": 90.873, "latency": 5.006, "stderr": 1.728, "cost_per_test": 0.000492, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-v3p2": { "accuracy": 90.873, "latency": 8.966, "stderr": 1.768, "cost_per_test": 0.00015, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 90.873, "latency": 14.087, "stderr": 1.728, "cost_per_test": 0.014593, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.5": { "accuracy": 90.836, "latency": 89.549, "stderr": 1.791, "cost_per_test": 0.006319, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 90.691, "latency": 6.731, "stderr": 1.789, "cost_per_test": 0.002941, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 90.436, "latency": 10.068, "stderr": 1.84, "cost_per_test": 0.000798, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4o-2024-11-20": { "accuracy": 90.364, "latency": 3.982, "stderr": 1.826, "cost_per_test": 0.004323, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 90.364, "latency": 6.56, "stderr": 1.836, "cost_per_test": 0.000509, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "kimi/kimi-k2-thinking": { "accuracy": 90.146, "latency": 48.41, "stderr": 1.83, "cost_per_test": 0.000891, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 89.854, "latency": 3.413, "stderr": 1.8, "cost_per_test": 0.001015, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 89.818, "latency": 5.733, "stderr": 1.787, "cost_per_test": 0.001019, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "zai/glm-4.6": { "accuracy": 89.746, "latency": 34.307, "stderr": 1.803, "cost_per_test": 0.004302, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 89.527, "latency": 1.465, "stderr": 1.894, "cost_per_test": 0.000212, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 89.527, "latency": 13.501, "stderr": 1.892, "cost_per_test": 0.000518, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/o1-2024-12-17": { "accuracy": 89.309, "latency": 11.207, "stderr": 1.779, "cost_per_test": 0.057248, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 89.309, "latency": 22.775, "stderr": 1.892, "cost_per_test": 0.001888, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "google/gemini-1.5-pro-002": { "accuracy": 89.2, "latency": 2.798, "stderr": 1.921, "cost_per_test": 0.001014, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.0-flash-001": { "accuracy": 89.018, "latency": 1.536, "stderr": 1.877, "cost_per_test": 0.000102, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/gpt-oss-20b": { "accuracy": 89.018, "latency": 5.791, "stderr": 1.89, "cost_per_test": 0.000453, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 88.4, "latency": 4.568, "stderr": 1.928, "cost_per_test": 0.000147, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "zai/glm-4.7": { "accuracy": 88.182, "latency": 63.175, "stderr": 1.828, "cost_per_test": 0.004315, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 88.0, "latency": 1.811, "stderr": 1.999, "cost_per_test": 0.000338, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 87.964, "latency": 3.499, "stderr": 1.993, "cost_per_test": 0.000147, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "minimax/MiniMax-M2.1": { "accuracy": 87.854, "latency": 13.862, "stderr": 1.827, "cost_per_test": 0.001335, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 87.782, "latency": 2.748, "stderr": 1.928, "cost_per_test": 0.000394, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 87.673, "latency": 2.19, "stderr": 1.921, "cost_per_test": 0.001684, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 87.564, "latency": 3.683, "stderr": 2.036, "cost_per_test": 0.000147, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2411": { "accuracy": 87.236, "latency": 8.041, "stderr": 2.045, "cost_per_test": 0.002954, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-1.5-flash-002": { "accuracy": 86.582, "latency": 1.413, "stderr": 2.126, "cost_per_test": 0.000064, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/magistral-small-2509": { "accuracy": 86.254, "latency": 8.038, "stderr": 2.065, "cost_per_test": 0.001465, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 86.182, "latency": 4.033, "stderr": 2.133, "cost_per_test": 0.000187, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-2-1212": { "accuracy": 86.146, "latency": 6.272, "stderr": 2.001, "cost_per_test": 0.003759, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 86.036, "latency": 28.224, "stderr": 2.037, "cost_per_test": 0.0004, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "cohere/command-a-03-2025": { "accuracy": 85.709, "latency": 8.36, "stderr": 2.016, "cost_per_test": 0.003732, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "mistralai/mistral-large-2512": { "accuracy": 85.418, "latency": 7.622, "stderr": 1.787, "cost_per_test": 0.000472, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 84.618, "latency": 3.738, "stderr": 2.203, "cost_per_test": 0.001116, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-small-2503": { "accuracy": 84.218, "latency": 3.87, "stderr": 2.21, "cost_per_test": 0.000095, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2402": { "accuracy": 83.964, "latency": 2.953, "stderr": 2.238, "cost_per_test": 0.000208, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2509": { "accuracy": 74.618, "latency": 11.933, "stderr": 2.549, "cost_per_test": 0.005709, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "ai21labs/jamba-large-1.6": { "accuracy": 71.236, "latency": 9.862, "stderr": 2.6, "cost_per_test": 0.003816, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 69.273, "latency": 1.458, "stderr": 2.799, "cost_per_test": 0.000129, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 41.709, "latency": 3.927, "stderr": 2.871, "cost_per_test": 0.000279, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_en": { "minimax/MiniMax-M2.1": { "accuracy": 100.0, "latency": 12.127, "stderr": 0.0, "cost_per_test": 0.001154, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 99.6, "latency": 3.906, "stderr": 0.399, "cost_per_test": 0.016598, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-medium-2505": { "accuracy": 99.2, "latency": 3.566, "stderr": 0.563, "cost_per_test": 0.000522, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/o4-mini-2025-04-16": { "accuracy": 99.2, "latency": 5.717, "stderr": 0.563, "cost_per_test": 0.001956, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 99.2, "latency": 7.883, "stderr": 0.563, "cost_per_test": 0.039987, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.6": { "accuracy": 99.2, "latency": 14.244, "stderr": 0.563, "cost_per_test": 0.001528, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 98.8, "latency": 4.456, "stderr": 0.689, "cost_per_test": 0.003618, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 98.8, "latency": 6.378, "stderr": 0.689, "cost_per_test": 0.003675, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 98.8, "latency": 7.18, "stderr": 0.689, "cost_per_test": 0.007283, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 98.8, "latency": 9.874, "stderr": 0.689, "cost_per_test": 0.006334, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-pro-preview": { "accuracy": 98.8, "latency": 12.222, "stderr": 0.689, "cost_per_test": 0.016642, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3-flash-preview": { "accuracy": 98.8, "latency": 12.686, "stderr": 0.689, "cost_per_test": 0.006511, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3-max-preview": { "accuracy": 98.8, "latency": 12.959, "stderr": 0.689, "cost_per_test": 0.00137, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 98.4, "latency": 2.54, "stderr": 0.794, "cost_per_test": 0.000188, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "mistralai/mistral-large-2512": { "accuracy": 98.4, "latency": 5.293, "stderr": 0.794, "cost_per_test": 0.000383, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 98.4, "latency": 5.56, "stderr": 0.794, "cost_per_test": 0.003854, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "alibaba/qwen3-max": { "accuracy": 98.4, "latency": 5.91, "stderr": 0.794, "cost_per_test": 0.001448, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 98.4, "latency": 6.352, "stderr": 0.794, "cost_per_test": 0.001688, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3-0324": { "accuracy": 98.4, "latency": 8.078, "stderr": 0.794, "cost_per_test": 0.000269, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-mini-2025-01-31": { "accuracy": 98.4, "latency": 8.934, "stderr": 0.794, "cost_per_test": 0.004584, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 98.4, "latency": 10.649, "stderr": 0.794, "cost_per_test": 0.008347, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.7": { "accuracy": 98.4, "latency": 27.636, "stderr": 0.794, "cost_per_test": 0.002477, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 98.0, "latency": 2.726, "stderr": 0.885, "cost_per_test": 0.002752, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 98.0, "latency": 2.866, "stderr": 0.885, "cost_per_test": 0.003339, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 98.0, "latency": 3.644, "stderr": 0.885, "cost_per_test": 0.000743, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/gpt-oss-120b": { "accuracy": 98.0, "latency": 3.648, "stderr": 0.885, "cost_per_test": 0.000417, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "mistralai/magistral-small-2509": { "accuracy": 98.0, "latency": 4.048, "stderr": 0.885, "cost_per_test": 0.001074, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-v3p2": { "accuracy": 98.0, "latency": 7.538, "stderr": 0.885, "cost_per_test": 0.000119, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 98.0, "latency": 7.655, "stderr": 0.885, "cost_per_test": 0.016726, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 98.0, "latency": 7.734, "stderr": 0.885, "cost_per_test": 0.000611, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "fireworks/deepseek-r1": { "accuracy": 98.0, "latency": 9.042, "stderr": 0.885, "cost_per_test": 0.003612, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 98.0, "latency": 14.538, "stderr": 0.885, "cost_per_test": 0.016748, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/gpt-oss-20b": { "accuracy": 97.6, "latency": 2.814, "stderr": 0.968, "cost_per_test": 0.000225, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "cohere/command-a-03-2025": { "accuracy": 97.6, "latency": 4.648, "stderr": 0.968, "cost_per_test": 0.002382, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "openai/o3-2025-04-16": { "accuracy": 97.6, "latency": 4.836, "stderr": 0.968, "cost_per_test": 0.002655, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 97.6, "latency": 5.333, "stderr": 0.968, "cost_per_test": 0.015887, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 97.6, "latency": 9.578, "stderr": 0.968, "cost_per_test": 0.008759, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 97.6, "latency": 9.956, "stderr": 0.968, "cost_per_test": 0.041861, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3": { "accuracy": 97.6, "latency": 11.282, "stderr": 0.968, "cost_per_test": 0.000255, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 97.6, "latency": 15.385, "stderr": 0.968, "cost_per_test": 0.001373, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 97.6, "latency": 15.602, "stderr": 0.968, "cost_per_test": 0.036604, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 97.6, "latency": 23.251, "stderr": 0.968, "cost_per_test": 0.000326, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 97.6, "latency": 31.272, "stderr": 0.968, "cost_per_test": 0.001036, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 97.2, "latency": 1.313, "stderr": 1.043, "cost_per_test": 0.000256, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 97.2, "latency": 2.247, "stderr": 1.043, "cost_per_test": 0.000726, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-3": { "accuracy": 97.2, "latency": 6.223, "stderr": 1.043, "cost_per_test": 0.005651, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 97.2, "latency": 6.877, "stderr": 1.043, "cost_per_test": 0.0028, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 96.8, "latency": 3.136, "stderr": 1.113, "cost_per_test": 0.000115, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-2-1212": { "accuracy": 96.8, "latency": 3.528, "stderr": 1.113, "cost_per_test": 0.002058, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 96.4, "latency": 3.623, "stderr": 1.178, "cost_per_test": 0.000133, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/mistral-large-2411": { "accuracy": 96.4, "latency": 4.49, "stderr": 1.178, "cost_per_test": 0.001476, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2509": { "accuracy": 96.4, "latency": 9.566, "stderr": 1.178, "cost_per_test": 0.004643, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "kimi/kimi-k2-thinking": { "accuracy": 96.4, "latency": 21.99, "stderr": 1.178, "cost_per_test": 0.000508, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "mistralai/mistral-small-2402": { "accuracy": 96.0, "latency": 2.018, "stderr": 1.239, "cost_per_test": 0.000132, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 96.0, "latency": 2.11, "stderr": 1.239, "cost_per_test": 0.000089, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 96.0, "latency": 3.399, "stderr": 1.239, "cost_per_test": 0.000334, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4o-2024-11-20": { "accuracy": 96.0, "latency": 3.558, "stderr": 1.239, "cost_per_test": 0.003573, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.5": { "accuracy": 96.0, "latency": 33.346, "stderr": 1.239, "cost_per_test": 0.003478, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 95.6, "latency": 1.1, "stderr": 1.297, "cost_per_test": 0.000146, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.0-flash-001": { "accuracy": 95.6, "latency": 1.38, "stderr": 1.297, "cost_per_test": 0.000089, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 95.6, "latency": 1.58, "stderr": 1.297, "cost_per_test": 0.001176, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 95.6, "latency": 1.59, "stderr": 1.297, "cost_per_test": 0.000275, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 95.6, "latency": 1.957, "stderr": 1.297, "cost_per_test": 0.000377, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-small-2503": { "accuracy": 95.6, "latency": 2.665, "stderr": 1.297, "cost_per_test": 0.000066, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 95.6, "latency": 2.883, "stderr": 1.297, "cost_per_test": 0.000819, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 95.2, "latency": 1.452, "stderr": 1.352, "cost_per_test": 0.000228, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 95.2, "latency": 6.779, "stderr": 1.352, "cost_per_test": 0.000298, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-pro-002": { "accuracy": 94.8, "latency": 1.961, "stderr": 1.404, "cost_per_test": 0.000732, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 94.8, "latency": 3.59, "stderr": 1.404, "cost_per_test": 0.000185, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-0709": { "accuracy": 94.4, "latency": 36.431, "stderr": 1.454, "cost_per_test": 0.019169, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-flash-002": { "accuracy": 94.0, "latency": 1.09, "stderr": 1.502, "cost_per_test": 0.000046, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 92.0, "latency": 6.89, "stderr": 1.716, "cost_per_test": 0.000539, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "ai21labs/jamba-large-1.6": { "accuracy": 88.4, "latency": 3.968, "stderr": 2.025, "cost_per_test": 0.001623, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 71.6, "latency": 0.887, "stderr": 2.852, "cost_per_test": 0.000075, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 57.6, "latency": 1.697, "stderr": 3.126, "cost_per_test": 0.000108, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_ja": { "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 93.6, "latency": 11.13, "stderr": 1.548, "cost_per_test": 0.059433, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 93.6, "latency": 13.476, "stderr": 1.548, "cost_per_test": 0.01864, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 93.2, "latency": 10.355, "stderr": 1.592, "cost_per_test": 0.008581, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 92.4, "latency": 4.912, "stderr": 1.676, "cost_per_test": 0.021863, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 92.4, "latency": 11.358, "stderr": 1.676, "cost_per_test": 0.013792, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-flash-preview": { "accuracy": 92.0, "latency": 14.045, "stderr": 1.716, "cost_per_test": 0.007059, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 92.0, "latency": 14.729, "stderr": 1.716, "cost_per_test": 0.051547, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 92.0, "latency": 32.849, "stderr": 1.716, "cost_per_test": 0.021638, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/o3-2025-04-16": { "accuracy": 91.6, "latency": 8.646, "stderr": 1.754, "cost_per_test": 0.004752, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-r1": { "accuracy": 91.6, "latency": 9.785, "stderr": 1.754, "cost_per_test": 0.003793, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 91.2, "latency": 6.919, "stderr": 1.792, "cost_per_test": 0.023354, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 91.2, "latency": 9.631, "stderr": 1.792, "cost_per_test": 0.010282, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 90.8, "latency": 8.6, "stderr": 1.828, "cost_per_test": 0.023178, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3": { "accuracy": 90.8, "latency": 9.899, "stderr": 1.828, "cost_per_test": 0.000296, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 90.8, "latency": 9.925, "stderr": 1.828, "cost_per_test": 0.006973, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/qwen3-235b-a22b": { "accuracy": 90.8, "latency": 30.303, "stderr": 1.828, "cost_per_test": 0.001099, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "mistralai/mistral-medium-2505": { "accuracy": 90.4, "latency": 7.831, "stderr": 1.863, "cost_per_test": 0.001033, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "alibaba/qwen3-max": { "accuracy": 90.4, "latency": 10.261, "stderr": 1.863, "cost_per_test": 0.00246, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/o3-mini-2025-01-31": { "accuracy": 90.4, "latency": 13.462, "stderr": 1.863, "cost_per_test": 0.008694, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 90.0, "latency": 8.589, "stderr": 1.897, "cost_per_test": 0.003033, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-large-2512": { "accuracy": 90.0, "latency": 9.533, "stderr": 1.897, "cost_per_test": 0.000579, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/o1-2024-12-17": { "accuracy": 90.0, "latency": 13.163, "stderr": 1.897, "cost_per_test": 0.079831, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3-0324": { "accuracy": 90.0, "latency": 14.814, "stderr": 1.897, "cost_per_test": 0.000423, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 90.0, "latency": 15.318, "stderr": 1.897, "cost_per_test": 0.015829, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "minimax/MiniMax-M2.1": { "accuracy": 90.0, "latency": 17.872, "stderr": 1.897, "cost_per_test": 0.00172, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "zai/glm-4.5": { "accuracy": 90.0, "latency": 41.805, "stderr": 1.897, "cost_per_test": 0.004081, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 89.6, "latency": 3.393, "stderr": 1.931, "cost_per_test": 0.003605, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 89.6, "latency": 3.424, "stderr": 1.931, "cost_per_test": 0.000962, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3p2": { "accuracy": 89.6, "latency": 9.139, "stderr": 1.931, "cost_per_test": 0.00015, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 89.6, "latency": 12.224, "stderr": 1.931, "cost_per_test": 0.001072, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 89.6, "latency": 23.506, "stderr": 1.931, "cost_per_test": 0.027165, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.6": { "accuracy": 89.6, "latency": 29.872, "stderr": 1.931, "cost_per_test": 0.003779, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "zai/glm-4.7": { "accuracy": 89.6, "latency": 43.372, "stderr": 1.931, "cost_per_test": 0.003684, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 89.2, "latency": 3.355, "stderr": 1.963, "cost_per_test": 0.000519, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 89.2, "latency": 5.207, "stderr": 1.963, "cost_per_test": 0.000975, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/gpt-oss-20b": { "accuracy": 89.2, "latency": 5.378, "stderr": 1.963, "cost_per_test": 0.000438, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 89.2, "latency": 5.384, "stderr": 1.963, "cost_per_test": 0.005079, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 89.2, "latency": 13.01, "stderr": 1.963, "cost_per_test": 0.004011, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max-preview": { "accuracy": 89.2, "latency": 19.42, "stderr": 1.963, "cost_per_test": 0.001918, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 88.8, "latency": 2.25, "stderr": 1.995, "cost_per_test": 0.000422, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 88.8, "latency": 4.217, "stderr": 1.995, "cost_per_test": 0.004829, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 88.8, "latency": 5.884, "stderr": 1.995, "cost_per_test": 0.000122, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/gpt-oss-120b": { "accuracy": 88.8, "latency": 6.102, "stderr": 1.995, "cost_per_test": 0.000721, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4-0709": { "accuracy": 88.8, "latency": 48.936, "stderr": 1.995, "cost_per_test": 0.02108, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 88.4, "latency": 27.152, "stderr": 2.025, "cost_per_test": 0.00038, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-2-1212": { "accuracy": 88.0, "latency": 5.667, "stderr": 2.055, "cost_per_test": 0.003255, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 87.6, "latency": 1.629, "stderr": 2.084, "cost_per_test": 0.000207, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 87.6, "latency": 12.124, "stderr": 2.084, "cost_per_test": 0.00048, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.0-flash-001": { "accuracy": 87.2, "latency": 1.366, "stderr": 2.113, "cost_per_test": 0.000076, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 87.2, "latency": 2.137, "stderr": 2.113, "cost_per_test": 0.000168, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 87.2, "latency": 2.287, "stderr": 2.113, "cost_per_test": 0.000346, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/magistral-small-2509": { "accuracy": 87.2, "latency": 8.618, "stderr": 2.113, "cost_per_test": 0.001437, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 87.2, "latency": 8.874, "stderr": 2.113, "cost_per_test": 0.003449, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 87.2, "latency": 10.958, "stderr": 2.113, "cost_per_test": 0.000432, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 87.2, "latency": 24.385, "stderr": 2.113, "cost_per_test": 0.002103, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 86.8, "latency": 2.064, "stderr": 2.141, "cost_per_test": 0.001731, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-1.5-pro-002": { "accuracy": 86.8, "latency": 2.776, "stderr": 2.141, "cost_per_test": 0.000956, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "cohere/command-a-03-2025": { "accuracy": 86.8, "latency": 4.279, "stderr": 2.141, "cost_per_test": 0.002281, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 86.8, "latency": 8.635, "stderr": 2.141, "cost_per_test": 0.00077, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-3": { "accuracy": 86.4, "latency": 5.354, "stderr": 2.168, "cost_per_test": 0.005902, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4o-2024-11-20": { "accuracy": 86.0, "latency": 4.522, "stderr": 2.194, "cost_per_test": 0.005571, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2-thinking": { "accuracy": 85.6, "latency": 40.689, "stderr": 2.22, "cost_per_test": 0.000766, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 85.2, "latency": 3.418, "stderr": 2.246, "cost_per_test": 0.000137, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2411": { "accuracy": 85.2, "latency": 6.672, "stderr": 2.246, "cost_per_test": 0.002424, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 84.8, "latency": 2.867, "stderr": 2.271, "cost_per_test": 0.000284, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-flash-002": { "accuracy": 84.4, "latency": 1.288, "stderr": 2.295, "cost_per_test": 0.000056, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 84.0, "latency": 5.042, "stderr": 2.319, "cost_per_test": 0.000201, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 84.0, "latency": 14.445, "stderr": 2.319, "cost_per_test": 0.005964, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 83.6, "latency": 4.401, "stderr": 2.342, "cost_per_test": 0.000093, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2402": { "accuracy": 82.4, "latency": 2.885, "stderr": 2.408, "cost_per_test": 0.000197, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 81.6, "latency": 3.166, "stderr": 2.451, "cost_per_test": 0.000976, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 81.6, "latency": 3.335, "stderr": 2.451, "cost_per_test": 0.000128, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "ai21labs/jamba-large-1.6": { "accuracy": 74.4, "latency": 7.206, "stderr": 2.76, "cost_per_test": 0.003012, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 71.6, "latency": 1.8, "stderr": 2.852, "cost_per_test": 0.000177, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 36.4, "latency": 5.515, "stderr": 3.043, "cost_per_test": 0.000369, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_zh": { "anthropic/claude-opus-4-5-20251101": { "accuracy": 94.8, "latency": 4.528, "stderr": 1.404, "cost_per_test": 0.021248, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 94.8, "latency": 12.683, "stderr": 1.404, "cost_per_test": 0.017672, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3-flash-preview": { "accuracy": 94.8, "latency": 14.336, "stderr": 1.404, "cost_per_test": 0.007205, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 94.4, "latency": 8.58, "stderr": 1.454, "cost_per_test": 0.009357, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 94.4, "latency": 9.796, "stderr": 1.454, "cost_per_test": 0.05328, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 94.0, "latency": 9.524, "stderr": 1.502, "cost_per_test": 0.00754, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o3-mini-2025-01-31": { "accuracy": 94.0, "latency": 11.474, "stderr": 1.502, "cost_per_test": 0.006103, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 93.6, "latency": 6.653, "stderr": 1.548, "cost_per_test": 0.002363, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 93.6, "latency": 7.644, "stderr": 1.548, "cost_per_test": 0.005372, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 93.6, "latency": 8.679, "stderr": 1.548, "cost_per_test": 0.002404, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 93.6, "latency": 11.03, "stderr": 1.548, "cost_per_test": 0.044493, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 93.6, "latency": 15.227, "stderr": 1.548, "cost_per_test": 0.01113, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-medium-2505": { "accuracy": 93.2, "latency": 3.113, "stderr": 1.592, "cost_per_test": 0.000583, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 93.2, "latency": 5.35, "stderr": 1.592, "cost_per_test": 0.004939, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 93.2, "latency": 5.35, "stderr": 1.592, "cost_per_test": 0.004939, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 93.2, "latency": 6.543, "stderr": 1.592, "cost_per_test": 0.021548, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 93.2, "latency": 6.955, "stderr": 1.592, "cost_per_test": 0.004917, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max-preview": { "accuracy": 93.2, "latency": 14.511, "stderr": 1.592, "cost_per_test": 0.001438, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "kimi/kimi-k2-thinking": { "accuracy": 93.2, "latency": 43.048, "stderr": 1.592, "cost_per_test": 0.000779, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 92.8, "latency": 3.775, "stderr": 1.635, "cost_per_test": 0.004518, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 92.8, "latency": 28.044, "stderr": 1.635, "cost_per_test": 0.000389, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 92.4, "latency": 3.269, "stderr": 1.676, "cost_per_test": 0.003593, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-3": { "accuracy": 92.4, "latency": 3.98, "stderr": 1.676, "cost_per_test": 0.003922, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-120b": { "accuracy": 92.4, "latency": 4.631, "stderr": 1.676, "cost_per_test": 0.000529, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-2025-04-16": { "accuracy": 92.4, "latency": 5.39, "stderr": 1.676, "cost_per_test": 0.003373, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3p2": { "accuracy": 92.4, "latency": 7.792, "stderr": 1.676, "cost_per_test": 0.000122, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 92.4, "latency": 32.034, "stderr": 1.676, "cost_per_test": 0.000977, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "zai/glm-4.7": { "accuracy": 92.4, "latency": 32.413, "stderr": 1.676, "cost_per_test": 0.003232, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-0709": { "accuracy": 92.4, "latency": 40.934, "stderr": 1.676, "cost_per_test": 0.017593, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 92.0, "latency": 1.659, "stderr": 1.716, "cost_per_test": 0.001321, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 92.0, "latency": 2.517, "stderr": 1.716, "cost_per_test": 0.000412, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 92.0, "latency": 3.392, "stderr": 1.716, "cost_per_test": 0.000852, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 92.0, "latency": 4.584, "stderr": 1.716, "cost_per_test": 0.000304, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 92.0, "latency": 5.836, "stderr": 1.716, "cost_per_test": 0.000844, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3-max": { "accuracy": 92.0, "latency": 7.347, "stderr": 1.716, "cost_per_test": 0.001852, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/o1-2024-12-17": { "accuracy": 92.0, "latency": 8.595, "stderr": 1.716, "cost_per_test": 0.045393, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3-0324": { "accuracy": 92.0, "latency": 30.025, "stderr": 1.716, "cost_per_test": 0.000404, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 91.6, "latency": 2.333, "stderr": 1.754, "cost_per_test": 0.000182, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 91.6, "latency": 3.518, "stderr": 1.754, "cost_per_test": 0.003896, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 91.6, "latency": 6.182, "stderr": 1.754, "cost_per_test": 0.002637, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-large-2512": { "accuracy": 91.6, "latency": 6.266, "stderr": 1.754, "cost_per_test": 0.000468, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-opus-4-20250514": { "accuracy": 91.6, "latency": 8.642, "stderr": 1.754, "cost_per_test": 0.021931, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-r1": { "accuracy": 91.6, "latency": 12.744, "stderr": 1.754, "cost_per_test": 0.004962, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/deepseek-v3": { "accuracy": 91.6, "latency": 18.07, "stderr": 1.754, "cost_per_test": 0.000262, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 91.6, "latency": 18.297, "stderr": 1.754, "cost_per_test": 0.001439, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 91.2, "latency": 5.789, "stderr": 1.792, "cost_per_test": 0.000549, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 91.2, "latency": 8.268, "stderr": 1.792, "cost_per_test": 0.000321, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 91.2, "latency": 19.462, "stderr": 1.792, "cost_per_test": 0.022222, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.6": { "accuracy": 91.2, "latency": 28.167, "stderr": 1.792, "cost_per_test": 0.003112, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.0-flash-001": { "accuracy": 90.8, "latency": 1.382, "stderr": 1.828, "cost_per_test": 0.00008, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/gpt-oss-20b": { "accuracy": 90.8, "latency": 5.207, "stderr": 1.828, "cost_per_test": 0.000394, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "mistralai/mistral-large-2411": { "accuracy": 90.8, "latency": 6.333, "stderr": 1.828, "cost_per_test": 0.002129, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 90.8, "latency": 23.204, "stderr": 1.828, "cost_per_test": 0.001768, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.5": { "accuracy": 90.8, "latency": 38.728, "stderr": 1.828, "cost_per_test": 0.002939, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "cohere/command-a-03-2025": { "accuracy": 90.4, "latency": 9.789, "stderr": 1.863, "cost_per_test": 0.002222, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "minimax/MiniMax-M2.1": { "accuracy": 90.4, "latency": 15.983, "stderr": 1.863, "cost_per_test": 0.001426, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "mistralai/magistral-small-2509": { "accuracy": 90.0, "latency": 5.768, "stderr": 1.897, "cost_per_test": 0.001412, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 89.6, "latency": 1.44, "stderr": 1.931, "cost_per_test": 0.000182, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 89.6, "latency": 4.046, "stderr": 1.931, "cost_per_test": 0.000356, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-2-1212": { "accuracy": 89.6, "latency": 4.409, "stderr": 1.931, "cost_per_test": 0.002436, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 89.2, "latency": 1.201, "stderr": 1.963, "cost_per_test": 0.00025, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 89.2, "latency": 1.734, "stderr": 1.963, "cost_per_test": 0.000307, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 89.2, "latency": 2.804, "stderr": 1.963, "cost_per_test": 0.000123, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/mistral-small-2503": { "accuracy": 89.2, "latency": 3.922, "stderr": 1.963, "cost_per_test": 0.000089, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 88.8, "latency": 2.594, "stderr": 1.995, "cost_per_test": 0.000109, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-pro-002": { "accuracy": 88.4, "latency": 2.486, "stderr": 2.025, "cost_per_test": 0.000854, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 88.0, "latency": 4.266, "stderr": 2.055, "cost_per_test": 0.000116, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-1.5-flash-002": { "accuracy": 87.6, "latency": 1.233, "stderr": 2.084, "cost_per_test": 0.000052, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 86.8, "latency": 3.505, "stderr": 2.141, "cost_per_test": 0.001066, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-small-2402": { "accuracy": 86.0, "latency": 3.07, "stderr": 2.194, "cost_per_test": 0.000212, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 86.0, "latency": 3.807, "stderr": 2.194, "cost_per_test": 0.000175, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 83.6, "latency": 1.065, "stderr": 2.342, "cost_per_test": 0.000095, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-large-1.6": { "accuracy": 80.0, "latency": 5.837, "stderr": 2.53, "cost_per_test": 0.002293, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "mistralai/magistral-medium-2509": { "accuracy": 76.0, "latency": 9.485, "stderr": 2.701, "cost_per_test": 0.005932, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 52.8, "latency": 2.512, "stderr": 3.157, "cost_per_test": 0.000173, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_bn": { "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 96.8, "latency": 15.051, "stderr": 1.113, "cost_per_test": 0.091543, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 95.2, "latency": 7.31, "stderr": 1.352, "cost_per_test": 0.041163, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 94.8, "latency": 11.204, "stderr": 1.404, "cost_per_test": 0.010167, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 94.8, "latency": 12.791, "stderr": 1.404, "cost_per_test": 0.015452, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 94.8, "latency": 15.858, "stderr": 1.404, "cost_per_test": 0.012099, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 94.0, "latency": 10.438, "stderr": 1.502, "cost_per_test": 0.003051, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-20250514": { "accuracy": 93.6, "latency": 11.348, "stderr": 1.548, "cost_per_test": 0.040106, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 93.2, "latency": 8.319, "stderr": 1.592, "cost_per_test": 0.006064, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 93.2, "latency": 10.183, "stderr": 1.592, "cost_per_test": 0.04033, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 93.2, "latency": 17.162, "stderr": 1.592, "cost_per_test": 0.074349, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/gpt-oss-120b": { "accuracy": 92.8, "latency": 6.584, "stderr": 1.635, "cost_per_test": 0.000754, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o4-mini-2025-04-16": { "accuracy": 92.8, "latency": 7.486, "stderr": 1.635, "cost_per_test": 0.002825, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 92.4, "latency": 3.963, "stderr": 1.676, "cost_per_test": 0.000307, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 92.4, "latency": 10.03, "stderr": 1.676, "cost_per_test": 0.007339, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3": { "accuracy": 91.6, "latency": 13.624, "stderr": 1.754, "cost_per_test": 0.000375, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 91.2, "latency": 7.582, "stderr": 1.792, "cost_per_test": 0.008156, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "alibaba/qwen3-max-preview": { "accuracy": 91.2, "latency": 32.312, "stderr": 1.792, "cost_per_test": 0.004158, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "google/gemini-1.5-pro-002": { "accuracy": 90.4, "latency": 3.331, "stderr": 1.863, "cost_per_test": 0.001408, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4o-2024-11-20": { "accuracy": 90.4, "latency": 5.405, "stderr": 1.863, "cost_per_test": 0.004795, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 90.4, "latency": 5.762, "stderr": 1.863, "cost_per_test": 0.005766, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 90.4, "latency": 6.581, "stderr": 1.863, "cost_per_test": 0.007968, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 90.4, "latency": 12.881, "stderr": 1.863, "cost_per_test": 0.017925, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3-0324": { "accuracy": 90.4, "latency": 15.573, "stderr": 1.863, "cost_per_test": 0.000393, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-mini-2025-01-31": { "accuracy": 90.0, "latency": 12.15, "stderr": 1.897, "cost_per_test": 0.007493, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 89.6, "latency": 6.816, "stderr": 1.931, "cost_per_test": 0.003198, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max": { "accuracy": 89.6, "latency": 23.778, "stderr": 1.931, "cost_per_test": 0.005813, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 89.6, "latency": 27.591, "stderr": 1.931, "cost_per_test": 0.033389, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.5": { "accuracy": 89.6, "latency": 177.154, "stderr": 1.931, "cost_per_test": 0.012559, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "mistralai/mistral-medium-2505": { "accuracy": 89.2, "latency": 5.31, "stderr": 1.963, "cost_per_test": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 88.8, "latency": 11.998, "stderr": 1.995, "cost_per_test": 0.001078, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 88.4, "latency": 6.591, "stderr": 2.025, "cost_per_test": 0.000691, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-r1": { "accuracy": 88.4, "latency": 9.908, "stderr": 2.025, "cost_per_test": 0.003897, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/qwen3-235b-a22b": { "accuracy": 88.4, "latency": 39.699, "stderr": 2.025, "cost_per_test": 0.001438, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 88.0, "latency": 5.073, "stderr": 2.055, "cost_per_test": 0.001021, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 87.6, "latency": 18.793, "stderr": 2.084, "cost_per_test": 0.001236, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 87.6, "latency": 20.59, "stderr": 2.084, "cost_per_test": 0.000735, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-0709": { "accuracy": 87.6, "latency": 165.545, "stderr": 2.084, "cost_per_test": 0.034715, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2411": { "accuracy": 86.8, "latency": 12.402, "stderr": 2.141, "cost_per_test": 0.004614, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 86.4, "latency": 5.596, "stderr": 2.168, "cost_per_test": 0.000906, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "kimi/kimi-k2-thinking": { "accuracy": 86.4, "latency": 66.997, "stderr": 2.168, "cost_per_test": 0.00119, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "openai/o3-2025-04-16": { "accuracy": 86.0, "latency": 6.609, "stderr": 2.194, "cost_per_test": 0.003954, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 86.0, "latency": 24.014, "stderr": 2.194, "cost_per_test": 0.002036, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "google/gemini-1.5-flash-002": { "accuracy": 85.6, "latency": 1.932, "stderr": 2.22, "cost_per_test": 0.000095, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "cohere/command-a-03-2025": { "accuracy": 84.4, "latency": 14.944, "stderr": 2.295, "cost_per_test": 0.006158, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 84.4, "latency": 25.754, "stderr": 2.295, "cost_per_test": 0.000363, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "fireworks/gpt-oss-20b": { "accuracy": 84.0, "latency": 7.891, "stderr": 2.319, "cost_per_test": 0.000548, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-3": { "accuracy": 82.8, "latency": 8.702, "stderr": 2.387, "cost_per_test": 0.009055, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 82.4, "latency": 1.667, "stderr": 2.408, "cost_per_test": 0.000343, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 81.6, "latency": 4.997, "stderr": 2.451, "cost_per_test": 0.001422, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 81.2, "latency": 3.574, "stderr": 2.471, "cost_per_test": 0.000143, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "fireworks/deepseek-v3p2": { "accuracy": 81.2, "latency": 8.965, "stderr": 2.471, "cost_per_test": 0.000142, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "mistralai/mistral-small-2402": { "accuracy": 79.6, "latency": 3.744, "stderr": 2.549, "cost_per_test": 0.00026, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 79.6, "latency": 6.105, "stderr": 2.549, "cost_per_test": 0.000242, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 79.2, "latency": 3.735, "stderr": 2.567, "cost_per_test": 0.000199, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/magistral-small-2509": { "accuracy": 79.2, "latency": 7.496, "stderr": 2.567, "cost_per_test": 0.001608, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 78.4, "latency": 2.409, "stderr": 2.603, "cost_per_test": 0.000531, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-small-2503": { "accuracy": 78.4, "latency": 4.393, "stderr": 2.603, "cost_per_test": 0.000117, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-3-flash-preview": { "accuracy": 78.4, "latency": 18.193, "stderr": 2.603, "cost_per_test": 0.009039, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 76.0, "latency": 16.555, "stderr": 2.701, "cost_per_test": 0.018237, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.6": { "accuracy": 74.4, "latency": 54.508, "stderr": 2.76, "cost_per_test": 0.006962, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 72.4, "latency": 3.567, "stderr": 2.827, "cost_per_test": 0.001565, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 71.6, "latency": 5.77, "stderr": 2.852, "cost_per_test": 0.001603, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 71.2, "latency": 4.638, "stderr": 2.864, "cost_per_test": 0.000228, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.0-flash-001": { "accuracy": 70.8, "latency": 1.512, "stderr": 2.876, "cost_per_test": 0.000133, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 67.2, "latency": 1.441, "stderr": 2.969, "cost_per_test": 0.000144, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-large-1.6": { "accuracy": 65.2, "latency": 18.962, "stderr": 3.013, "cost_per_test": 0.007558, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 63.2, "latency": 2.413, "stderr": 3.05, "cost_per_test": 0.000473, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o1-2024-12-17": { "accuracy": 60.8, "latency": 11.814, "stderr": 3.088, "cost_per_test": 0.056073, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 59.2, "latency": 3.408, "stderr": 3.108, "cost_per_test": 0.002781, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.1": { "accuracy": 58.8, "latency": 12.422, "stderr": 3.113, "cost_per_test": 0.001291, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "zai/glm-4.7": { "accuracy": 56.8, "latency": 91.061, "stderr": 3.133, "cost_per_test": 0.006955, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "mistralai/magistral-medium-2509": { "accuracy": 55.6, "latency": 13.054, "stderr": 3.142, "cost_per_test": 0.005785, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-2-1212": { "accuracy": 52.4, "latency": 8.366, "stderr": 3.159, "cost_per_test": 0.004885, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 30.4, "latency": 6.196, "stderr": 2.909, "cost_per_test": 0.000458, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "mistralai/mistral-large-2512": { "accuracy": 23.2, "latency": 10.349, "stderr": 2.67, "cost_per_test": 0.000601, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" } }, "mgsm_ru": { "openai/gpt-5.1-2025-11-13": { "accuracy": 97.2, "latency": 6.241, "stderr": 1.043, "cost_per_test": 0.004853, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 96.8, "latency": 8.752, "stderr": 1.113, "cost_per_test": 0.010349, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 96.8, "latency": 9.876, "stderr": 1.113, "cost_per_test": 0.055715, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.7": { "accuracy": 96.8, "latency": 58.874, "stderr": 1.113, "cost_per_test": 0.003145, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 96.4, "latency": 4.987, "stderr": 1.178, "cost_per_test": 0.025027, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 96.4, "latency": 12.327, "stderr": 1.178, "cost_per_test": 0.051543, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 96.4, "latency": 12.872, "stderr": 1.178, "cost_per_test": 0.018435, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 96.4, "latency": 18.78, "stderr": 1.178, "cost_per_test": 0.023222, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.0-flash-001": { "accuracy": 96.0, "latency": 1.558, "stderr": 1.239, "cost_per_test": 0.000093, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-20250514": { "accuracy": 96.0, "latency": 9.046, "stderr": 1.239, "cost_per_test": 0.025432, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 96.0, "latency": 11.772, "stderr": 1.239, "cost_per_test": 0.005978, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 96.0, "latency": 12.36, "stderr": 1.239, "cost_per_test": 0.013812, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 96.0, "latency": 12.966, "stderr": 1.239, "cost_per_test": 0.010027, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 95.6, "latency": 4.169, "stderr": 1.297, "cost_per_test": 0.005283, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-3": { "accuracy": 95.6, "latency": 4.392, "stderr": 1.297, "cost_per_test": 0.00478, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/o3-2025-04-16": { "accuracy": 95.6, "latency": 6.135, "stderr": 1.297, "cost_per_test": 0.003421, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 95.6, "latency": 6.146, "stderr": 1.297, "cost_per_test": 0.002825, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 95.6, "latency": 6.401, "stderr": 1.297, "cost_per_test": 0.002327, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max": { "accuracy": 95.6, "latency": 7.726, "stderr": 1.297, "cost_per_test": 0.002025, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/o1-2024-12-17": { "accuracy": 95.6, "latency": 9.194, "stderr": 1.297, "cost_per_test": 0.050378, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 95.6, "latency": 11.073, "stderr": 1.297, "cost_per_test": 0.00318, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "fireworks/qwen3-235b-a22b": { "accuracy": 95.6, "latency": 21.863, "stderr": 1.297, "cost_per_test": 0.000719, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 95.2, "latency": 2.611, "stderr": 1.352, "cost_per_test": 0.000364, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 95.2, "latency": 2.841, "stderr": 1.352, "cost_per_test": 0.00049, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-20b": { "accuracy": 95.2, "latency": 4.289, "stderr": 1.352, "cost_per_test": 0.000321, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/gpt-oss-120b": { "accuracy": 95.2, "latency": 6.436, "stderr": 1.352, "cost_per_test": 0.00055, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 95.2, "latency": 7.706, "stderr": 1.352, "cost_per_test": 0.025435, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "cohere/command-a-03-2025": { "accuracy": 95.2, "latency": 11.513, "stderr": 1.352, "cost_per_test": 0.002456, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 95.2, "latency": 11.783, "stderr": 1.352, "cost_per_test": 0.010474, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-medium-2505": { "accuracy": 94.8, "latency": 3.664, "stderr": 1.404, "cost_per_test": 0.000535, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 94.8, "latency": 5.205, "stderr": 1.404, "cost_per_test": 0.001046, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 94.8, "latency": 8.512, "stderr": 1.404, "cost_per_test": 0.00701, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.6": { "accuracy": 94.8, "latency": 27.281, "stderr": 1.404, "cost_per_test": 0.003506, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 94.4, "latency": 1.41, "stderr": 1.454, "cost_per_test": 0.000294, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 94.4, "latency": 3.424, "stderr": 1.454, "cost_per_test": 0.003801, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 94.4, "latency": 5.021, "stderr": 1.454, "cost_per_test": 0.004948, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-large-2512": { "accuracy": 94.4, "latency": 6.148, "stderr": 1.454, "cost_per_test": 0.000419, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "alibaba/qwen3-max-preview": { "accuracy": 94.4, "latency": 16.936, "stderr": 1.454, "cost_per_test": 0.001864, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "kimi/kimi-k2-thinking": { "accuracy": 94.4, "latency": 32.936, "stderr": 1.454, "cost_per_test": 0.000664, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "zai/glm-4.5": { "accuracy": 94.4, "latency": 34.597, "stderr": 1.454, "cost_per_test": 0.003411, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 94.0, "latency": 1.937, "stderr": 1.502, "cost_per_test": 0.001521, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 94.0, "latency": 2.55, "stderr": 1.502, "cost_per_test": 0.000199, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 94.0, "latency": 8.58, "stderr": 1.502, "cost_per_test": 0.000812, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-4-0709": { "accuracy": 94.0, "latency": 43.7, "stderr": 1.502, "cost_per_test": 0.019564, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 93.6, "latency": 1.496, "stderr": 1.548, "cost_per_test": 0.000182, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 93.6, "latency": 3.225, "stderr": 1.548, "cost_per_test": 0.00103, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4o-2024-11-20": { "accuracy": 93.6, "latency": 4.115, "stderr": 1.548, "cost_per_test": 0.004181, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 93.6, "latency": 6.071, "stderr": 1.548, "cost_per_test": 0.000556, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2411": { "accuracy": 93.6, "latency": 6.381, "stderr": 1.548, "cost_per_test": 0.002314, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 93.6, "latency": 10.677, "stderr": 1.548, "cost_per_test": 0.000401, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3-0324": { "accuracy": 93.6, "latency": 11.584, "stderr": 1.548, "cost_per_test": 0.000329, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/deepseek-v3": { "accuracy": 93.6, "latency": 16.81, "stderr": 1.548, "cost_per_test": 0.000327, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 93.6, "latency": 25.192, "stderr": 1.548, "cost_per_test": 0.002002, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "google/gemini-1.5-pro-002": { "accuracy": 93.2, "latency": 2.66, "stderr": 1.592, "cost_per_test": 0.000956, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 93.2, "latency": 4.472, "stderr": 1.592, "cost_per_test": 0.000134, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-r1": { "accuracy": 93.2, "latency": 8.022, "stderr": 1.592, "cost_per_test": 0.003213, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-mini-2025-01-31": { "accuracy": 93.2, "latency": 12.155, "stderr": 1.592, "cost_per_test": 0.007912, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.1": { "accuracy": 93.2, "latency": 13.097, "stderr": 1.592, "cost_per_test": 0.001262, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 92.8, "latency": 1.34, "stderr": 1.635, "cost_per_test": 0.000286, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 92.8, "latency": 3.039, "stderr": 1.635, "cost_per_test": 0.000135, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 92.8, "latency": 4.466, "stderr": 1.635, "cost_per_test": 0.000371, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 92.4, "latency": 3.01, "stderr": 1.676, "cost_per_test": 0.000124, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/magistral-small-2509": { "accuracy": 92.4, "latency": 4.996, "stderr": 1.676, "cost_per_test": 0.00125, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-2-1212": { "accuracy": 92.4, "latency": 5.009, "stderr": 1.676, "cost_per_test": 0.002874, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3p2": { "accuracy": 92.4, "latency": 8.35, "stderr": 1.676, "cost_per_test": 0.000141, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "mistralai/mistral-small-2503": { "accuracy": 91.2, "latency": 3.442, "stderr": 1.792, "cost_per_test": 0.000079, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-1.5-flash-002": { "accuracy": 90.4, "latency": 1.199, "stderr": 1.863, "cost_per_test": 0.000052, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 90.4, "latency": 3.727, "stderr": 1.863, "cost_per_test": 0.000175, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 90.0, "latency": 3.329, "stderr": 1.897, "cost_per_test": 0.001093, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-small-2402": { "accuracy": 89.6, "latency": 2.813, "stderr": 1.931, "cost_per_test": 0.000182, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "ai21labs/jamba-large-1.6": { "accuracy": 86.4, "latency": 9.809, "stderr": 2.168, "cost_per_test": 0.003836, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 71.6, "latency": 12.403, "stderr": 2.852, "cost_per_test": 0.000169, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 55.2, "latency": 3.058, "stderr": 3.145, "cost_per_test": 0.000226, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "mistralai/magistral-medium-2509": { "accuracy": 54.4, "latency": 12.627, "stderr": 3.15, "cost_per_test": 0.006632, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 45.6, "latency": 1.164, "stderr": 3.15, "cost_per_test": 0.000099, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" } }, "mgsm_de": { "anthropic/claude-opus-4-5-20251101": { "accuracy": 96.0, "latency": 4.886, "stderr": 1.239, "cost_per_test": 0.024285, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 95.2, "latency": 8.525, "stderr": 1.352, "cost_per_test": 0.009415, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 95.2, "latency": 9.447, "stderr": 1.352, "cost_per_test": 0.054599, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 95.2, "latency": 14.91, "stderr": 1.352, "cost_per_test": 0.0076, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/o4-mini-2025-04-16": { "accuracy": 94.4, "latency": 7.475, "stderr": 1.454, "cost_per_test": 0.002489, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 94.4, "latency": 11.522, "stderr": 1.454, "cost_per_test": 0.00274, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 94.4, "latency": 15.143, "stderr": 1.454, "cost_per_test": 0.009525, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 94.0, "latency": 9.446, "stderr": 1.502, "cost_per_test": 0.010058, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 94.0, "latency": 12.095, "stderr": 1.502, "cost_per_test": 0.049481, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 94.0, "latency": 12.34, "stderr": 1.502, "cost_per_test": 0.01775, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-medium-2505": { "accuracy": 93.6, "latency": 4.211, "stderr": 1.548, "cost_per_test": 0.000541, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 93.6, "latency": 7.833, "stderr": 1.548, "cost_per_test": 0.004707, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 93.6, "latency": 7.873, "stderr": 1.548, "cost_per_test": 0.005694, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 93.6, "latency": 8.377, "stderr": 1.548, "cost_per_test": 0.024738, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 93.6, "latency": 9.488, "stderr": 1.548, "cost_per_test": 0.045104, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o3-mini-2025-01-31": { "accuracy": 93.6, "latency": 11.05, "stderr": 1.548, "cost_per_test": 0.006496, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.1": { "accuracy": 93.6, "latency": 11.74, "stderr": 1.548, "cost_per_test": 0.001109, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "fireworks/qwen3-235b-a22b": { "accuracy": 93.6, "latency": 30.308, "stderr": 1.548, "cost_per_test": 0.001046, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 93.2, "latency": 2.207, "stderr": 1.592, "cost_per_test": 0.000524, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 93.2, "latency": 2.238, "stderr": 1.592, "cost_per_test": 0.000186, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 93.2, "latency": 3.255, "stderr": 1.592, "cost_per_test": 0.003356, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 93.2, "latency": 3.348, "stderr": 1.592, "cost_per_test": 0.000904, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/gpt-oss-20b": { "accuracy": 93.2, "latency": 4.301, "stderr": 1.592, "cost_per_test": 0.000366, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-2025-04-16": { "accuracy": 93.2, "latency": 6.493, "stderr": 1.592, "cost_per_test": 0.003612, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 93.2, "latency": 7.324, "stderr": 1.592, "cost_per_test": 0.024759, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "alibaba/qwen3-max": { "accuracy": 93.2, "latency": 8.557, "stderr": 1.592, "cost_per_test": 0.001979, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "fireworks/deepseek-v3": { "accuracy": 93.2, "latency": 21.066, "stderr": 1.592, "cost_per_test": 0.000354, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 93.2, "latency": 21.802, "stderr": 1.592, "cost_per_test": 0.026694, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.7": { "accuracy": 93.2, "latency": 55.969, "stderr": 1.592, "cost_per_test": 0.003262, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "cohere/command-a-03-2025": { "accuracy": 92.8, "latency": 4.368, "stderr": 1.635, "cost_per_test": 0.002258, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 92.8, "latency": 4.538, "stderr": 1.635, "cost_per_test": 0.000449, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-120b": { "accuracy": 92.8, "latency": 5.003, "stderr": 1.635, "cost_per_test": 0.000598, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 92.8, "latency": 11.95, "stderr": 1.635, "cost_per_test": 0.012954, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 92.8, "latency": 27.681, "stderr": 1.635, "cost_per_test": 0.000391, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "zai/glm-4.5": { "accuracy": 92.8, "latency": 39.613, "stderr": 1.635, "cost_per_test": 0.003943, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 92.4, "latency": 1.869, "stderr": 1.676, "cost_per_test": 0.001372, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 92.4, "latency": 2.029, "stderr": 1.676, "cost_per_test": 0.00038, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 92.4, "latency": 4.232, "stderr": 1.676, "cost_per_test": 0.005239, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 92.4, "latency": 5.886, "stderr": 1.676, "cost_per_test": 0.000871, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/magistral-small-2509": { "accuracy": 92.4, "latency": 6.26, "stderr": 1.676, "cost_per_test": 0.001442, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 92.4, "latency": 7.278, "stderr": 1.676, "cost_per_test": 0.000686, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-preview": { "accuracy": 92.4, "latency": 18.203, "stderr": 1.676, "cost_per_test": 0.001575, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "grok/grok-4-fast-reasoning": { "accuracy": 92.0, "latency": 3.262, "stderr": 1.716, "cost_per_test": 0.000556, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 92.0, "latency": 4.304, "stderr": 1.716, "cost_per_test": 0.000117, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4o-2024-11-20": { "accuracy": 92.0, "latency": 4.516, "stderr": 1.716, "cost_per_test": 0.004608, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3": { "accuracy": 92.0, "latency": 5.091, "stderr": 1.716, "cost_per_test": 0.004994, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 92.0, "latency": 5.21, "stderr": 1.716, "cost_per_test": 0.005118, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-large-2512": { "accuracy": 92.0, "latency": 5.931, "stderr": 1.716, "cost_per_test": 0.000408, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 92.0, "latency": 6.504, "stderr": 1.716, "cost_per_test": 0.002959, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-r1": { "accuracy": 92.0, "latency": 8.916, "stderr": 1.716, "cost_per_test": 0.003819, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "zai/glm-4.6": { "accuracy": 92.0, "latency": 29.96, "stderr": 1.716, "cost_per_test": 0.00336, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-0709": { "accuracy": 92.0, "latency": 160.431, "stderr": 1.716, "cost_per_test": 0.023082, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.0-flash-001": { "accuracy": 91.6, "latency": 1.497, "stderr": 1.754, "cost_per_test": 0.000088, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 91.2, "latency": 1.441, "stderr": 1.792, "cost_per_test": 0.0003, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-pro-002": { "accuracy": 91.2, "latency": 2.659, "stderr": 1.792, "cost_per_test": 0.000971, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 91.2, "latency": 3.637, "stderr": 1.792, "cost_per_test": 0.000148, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "kimi/kimi-k2-thinking": { "accuracy": 91.2, "latency": 30.451, "stderr": 1.792, "cost_per_test": 0.000669, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "fireworks/deepseek-v3-0324": { "accuracy": 91.2, "latency": 34.928, "stderr": 1.792, "cost_per_test": 0.000349, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 90.8, "latency": 1.346, "stderr": 1.828, "cost_per_test": 0.000175, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 90.8, "latency": 17.841, "stderr": 1.828, "cost_per_test": 0.001648, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 90.4, "latency": 3.386, "stderr": 1.863, "cost_per_test": 0.000998, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 90.4, "latency": 11.03, "stderr": 1.863, "cost_per_test": 0.000448, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-small-2503": { "accuracy": 90.0, "latency": 4.038, "stderr": 1.897, "cost_per_test": 0.000084, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-v3p2": { "accuracy": 90.0, "latency": 8.244, "stderr": 1.897, "cost_per_test": 0.000137, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-2-1212": { "accuracy": 89.6, "latency": 4.947, "stderr": 1.931, "cost_per_test": 0.002847, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 89.6, "latency": 8.658, "stderr": 1.931, "cost_per_test": 0.000765, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/mistral-large-2411": { "accuracy": 89.2, "latency": 6.346, "stderr": 1.963, "cost_per_test": 0.002167, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 88.8, "latency": 3.549, "stderr": 1.995, "cost_per_test": 0.000139, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "google/gemini-1.5-flash-002": { "accuracy": 88.4, "latency": 1.511, "stderr": 2.025, "cost_per_test": 0.000065, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-small-2402": { "accuracy": 88.0, "latency": 2.582, "stderr": 2.055, "cost_per_test": 0.000173, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 88.0, "latency": 4.14, "stderr": 2.055, "cost_per_test": 0.000206, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-large-1.6": { "accuracy": 84.0, "latency": 6.259, "stderr": 2.319, "cost_per_test": 0.002387, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 80.4, "latency": 1.319, "stderr": 2.511, "cost_per_test": 0.000106, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 79.6, "latency": 8.699, "stderr": 2.549, "cost_per_test": 0.004756, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 58.0, "latency": 2.604, "stderr": 3.122, "cost_per_test": 0.000159, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_fr": { "anthropic/claude-opus-4-1-20250805": { "accuracy": 93.2, "latency": 7.105, "stderr": 1.592, "cost_per_test": 0.022948, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 92.8, "latency": 7.36, "stderr": 1.635, "cost_per_test": 0.023164, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 92.0, "latency": 3.151, "stderr": 1.716, "cost_per_test": 0.003366, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 92.0, "latency": 12.387, "stderr": 1.716, "cost_per_test": 0.049382, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 91.6, "latency": 2.312, "stderr": 1.754, "cost_per_test": 0.00019, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "fireworks/deepseek-v3": { "accuracy": 91.6, "latency": 19.055, "stderr": 1.754, "cost_per_test": 0.000321, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 91.2, "latency": 4.845, "stderr": 1.792, "cost_per_test": 0.004687, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 90.8, "latency": 2.056, "stderr": 1.828, "cost_per_test": 0.000356, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-3": { "accuracy": 90.8, "latency": 4.564, "stderr": 1.828, "cost_per_test": 0.00425, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-r1": { "accuracy": 90.8, "latency": 9.078, "stderr": 1.828, "cost_per_test": 0.003632, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/deepseek-v3p2": { "accuracy": 90.4, "latency": 7.436, "stderr": 1.863, "cost_per_test": 0.000125, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "fireworks/deepseek-v3-0324": { "accuracy": 90.4, "latency": 33.321, "stderr": 1.863, "cost_per_test": 0.000324, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4-0709": { "accuracy": 90.4, "latency": 164.804, "stderr": 1.863, "cost_per_test": 0.024902, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 90.0, "latency": 4.858, "stderr": 1.897, "cost_per_test": 0.022666, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o4-mini-2025-04-16": { "accuracy": 90.0, "latency": 7.391, "stderr": 1.897, "cost_per_test": 0.002547, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 89.6, "latency": 8.78, "stderr": 1.931, "cost_per_test": 0.009224, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 88.8, "latency": 7.988, "stderr": 1.995, "cost_per_test": 0.000711, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 88.4, "latency": 18.879, "stderr": 2.025, "cost_per_test": 0.021357, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-large-2512": { "accuracy": 88.0, "latency": 4.771, "stderr": 2.055, "cost_per_test": 0.000365, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "alibaba/qwen3-max-preview": { "accuracy": 88.0, "latency": 14.843, "stderr": 2.055, "cost_per_test": 0.001545, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 87.6, "latency": 10.055, "stderr": 2.084, "cost_per_test": 0.05408, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/qwen3-235b-a22b": { "accuracy": 87.6, "latency": 37.275, "stderr": 2.084, "cost_per_test": 0.001105, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 87.2, "latency": 3.316, "stderr": 2.113, "cost_per_test": 0.000133, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "google/gemini-3-flash-preview": { "accuracy": 87.2, "latency": 13.663, "stderr": 2.113, "cost_per_test": 0.006964, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 86.8, "latency": 5.279, "stderr": 2.141, "cost_per_test": 0.000196, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 86.4, "latency": 1.419, "stderr": 2.168, "cost_per_test": 0.000314, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 86.4, "latency": 3.904, "stderr": 2.168, "cost_per_test": 0.004859, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "alibaba/qwen3-max": { "accuracy": 86.0, "latency": 7.213, "stderr": 2.194, "cost_per_test": 0.001761, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 86.0, "latency": 7.678, "stderr": 2.194, "cost_per_test": 0.000676, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 85.6, "latency": 1.232, "stderr": 2.22, "cost_per_test": 0.000167, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "cohere/command-a-03-2025": { "accuracy": 85.6, "latency": 4.639, "stderr": 2.22, "cost_per_test": 0.002533, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 85.6, "latency": 5.1, "stderr": 2.22, "cost_per_test": 0.000443, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 85.2, "latency": 20.814, "stderr": 2.246, "cost_per_test": 0.001677, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "grok/grok-2-1212": { "accuracy": 84.8, "latency": 4.484, "stderr": 2.271, "cost_per_test": 0.002594, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 84.8, "latency": 14.044, "stderr": 2.271, "cost_per_test": 0.014563, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.5": { "accuracy": 84.8, "latency": 48.63, "stderr": 2.271, "cost_per_test": 0.004542, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "fireworks/gpt-oss-20b": { "accuracy": 84.4, "latency": 5.049, "stderr": 2.295, "cost_per_test": 0.000405, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-2025-04-16": { "accuracy": 84.4, "latency": 6.333, "stderr": 2.295, "cost_per_test": 0.003652, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 84.4, "latency": 29.768, "stderr": 2.295, "cost_per_test": 0.000424, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 84.0, "latency": 2.795, "stderr": 2.319, "cost_per_test": 0.00087, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-large-2411": { "accuracy": 84.0, "latency": 6.305, "stderr": 2.319, "cost_per_test": 0.002178, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 84.0, "latency": 6.776, "stderr": 2.319, "cost_per_test": 0.002807, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 84.0, "latency": 7.846, "stderr": 2.319, "cost_per_test": 0.005555, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 84.0, "latency": 14.278, "stderr": 2.319, "cost_per_test": 0.068818, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 83.6, "latency": 1.27, "stderr": 2.342, "cost_per_test": 0.000279, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 83.6, "latency": 3.377, "stderr": 2.342, "cost_per_test": 0.000546, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/o3-mini-2025-01-31": { "accuracy": 83.6, "latency": 15.189, "stderr": 2.342, "cost_per_test": 0.008995, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.1": { "accuracy": 83.6, "latency": 18.392, "stderr": 2.342, "cost_per_test": 0.001738, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "google/gemini-2.0-flash-001": { "accuracy": 83.2, "latency": 1.874, "stderr": 2.364, "cost_per_test": 0.00012, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-medium-2505": { "accuracy": 83.2, "latency": 3.8, "stderr": 2.364, "cost_per_test": 0.000505, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/gpt-oss-120b": { "accuracy": 83.2, "latency": 5.527, "stderr": 2.364, "cost_per_test": 0.000658, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 82.8, "latency": 14.17, "stderr": 2.387, "cost_per_test": 0.004168, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.6": { "accuracy": 82.8, "latency": 27.413, "stderr": 2.387, "cost_per_test": 0.003435, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 82.4, "latency": 2.998, "stderr": 2.408, "cost_per_test": 0.000121, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 82.4, "latency": 3.326, "stderr": 2.408, "cost_per_test": 0.000808, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4o-2024-11-20": { "accuracy": 82.4, "latency": 3.747, "stderr": 2.408, "cost_per_test": 0.00354, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 82.4, "latency": 9.373, "stderr": 2.408, "cost_per_test": 0.006635, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2-thinking": { "accuracy": 82.4, "latency": 45.134, "stderr": 2.408, "cost_per_test": 0.000838, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 82.0, "latency": 11.581, "stderr": 2.43, "cost_per_test": 0.000474, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 82.0, "latency": 22.661, "stderr": 2.43, "cost_per_test": 0.017102, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.7": { "accuracy": 82.0, "latency": 44.622, "stderr": 2.43, "cost_per_test": 0.003198, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 81.6, "latency": 10.04, "stderr": 2.451, "cost_per_test": 0.011252, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-small-2402": { "accuracy": 80.8, "latency": 2.5, "stderr": 2.491, "cost_per_test": 0.000173, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 80.8, "latency": 5.743, "stderr": 2.491, "cost_per_test": 0.000833, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/magistral-small-2509": { "accuracy": 80.8, "latency": 6.056, "stderr": 2.491, "cost_per_test": 0.001436, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-3-pro-preview": { "accuracy": 80.8, "latency": 12.641, "stderr": 2.491, "cost_per_test": 0.017616, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 80.4, "latency": 1.687, "stderr": 2.511, "cost_per_test": 0.001301, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-large-1.6": { "accuracy": 80.4, "latency": 5.364, "stderr": 2.511, "cost_per_test": 0.001903, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 80.0, "latency": 4.211, "stderr": 2.53, "cost_per_test": 0.000126, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-small-2503": { "accuracy": 79.6, "latency": 3.627, "stderr": 2.549, "cost_per_test": 0.000084, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-1.5-flash-002": { "accuracy": 78.8, "latency": 1.173, "stderr": 2.585, "cost_per_test": 0.000052, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-1.5-pro-002": { "accuracy": 78.8, "latency": 3.567, "stderr": 2.585, "cost_per_test": 0.000934, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 76.8, "latency": 1.199, "stderr": 2.67, "cost_per_test": 0.000099, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 64.4, "latency": 10.531, "stderr": 3.028, "cost_per_test": 0.005809, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 56.8, "latency": 1.942, "stderr": 3.133, "cost_per_test": 0.000128, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_th": { "openai/gpt-5.2-2025-12-11": { "accuracy": 97.6, "latency": 9.728, "stderr": 0.968, "cost_per_test": 0.011857, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/gpt-oss-120b": { "accuracy": 96.4, "latency": 6.196, "stderr": 1.178, "cost_per_test": 0.000733, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 96.4, "latency": 14.818, "stderr": 1.178, "cost_per_test": 0.083074, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 96.4, "latency": 16.611, "stderr": 1.178, "cost_per_test": 0.008498, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-0709": { "accuracy": 96.0, "latency": 180.431, "stderr": 1.239, "cost_per_test": 0.025605, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 95.6, "latency": 6.336, "stderr": 1.297, "cost_per_test": 0.036327, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 95.2, "latency": 12.255, "stderr": 1.352, "cost_per_test": 0.017363, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5-2025-08-07": { "accuracy": 95.2, "latency": 25.197, "stderr": 1.352, "cost_per_test": 0.017551, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 94.8, "latency": 6.247, "stderr": 1.404, "cost_per_test": 0.000973, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/o3-2025-04-16": { "accuracy": 94.8, "latency": 6.717, "stderr": 1.404, "cost_per_test": 0.004014, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max-preview": { "accuracy": 94.8, "latency": 17.771, "stderr": 1.404, "cost_per_test": 0.002042, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "fireworks/qwen3-235b-a22b": { "accuracy": 94.8, "latency": 21.68, "stderr": 1.404, "cost_per_test": 0.000896, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 94.8, "latency": 24.092, "stderr": 1.404, "cost_per_test": 0.028326, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 94.4, "latency": 3.612, "stderr": 1.454, "cost_per_test": 0.000972, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/o4-mini-2025-04-16": { "accuracy": 94.4, "latency": 7.482, "stderr": 1.454, "cost_per_test": 0.002745, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 94.4, "latency": 8.603, "stderr": 1.454, "cost_per_test": 0.006629, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-r1": { "accuracy": 94.4, "latency": 8.704, "stderr": 1.454, "cost_per_test": 0.00371, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3-max": { "accuracy": 94.4, "latency": 8.977, "stderr": 1.454, "cost_per_test": 0.002407, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 94.4, "latency": 12.725, "stderr": 1.454, "cost_per_test": 0.015881, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o3-mini-2025-01-31": { "accuracy": 94.4, "latency": 12.778, "stderr": 1.454, "cost_per_test": 0.008565, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 94.4, "latency": 14.789, "stderr": 1.454, "cost_per_test": 0.004815, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 94.0, "latency": 4.971, "stderr": 1.502, "cost_per_test": 0.000859, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 94.0, "latency": 6.248, "stderr": 1.502, "cost_per_test": 0.008199, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 94.0, "latency": 10.268, "stderr": 1.502, "cost_per_test": 0.038907, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 94.0, "latency": 10.572, "stderr": 1.502, "cost_per_test": 0.038995, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 94.0, "latency": 12.863, "stderr": 1.502, "cost_per_test": 0.059025, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 94.0, "latency": 18.564, "stderr": 1.502, "cost_per_test": 0.07887, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 93.6, "latency": 2.164, "stderr": 1.548, "cost_per_test": 0.000415, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 93.6, "latency": 6.88, "stderr": 1.548, "cost_per_test": 0.007825, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.7": { "accuracy": 93.6, "latency": 123.392, "stderr": 1.548, "cost_per_test": 0.004954, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.0-flash-001": { "accuracy": 93.2, "latency": 1.352, "stderr": 1.592, "cost_per_test": 0.000091, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 93.2, "latency": 1.909, "stderr": 1.592, "cost_per_test": 0.001596, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 93.2, "latency": 4.588, "stderr": 1.592, "cost_per_test": 0.000137, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-3": { "accuracy": 93.2, "latency": 5.966, "stderr": 1.592, "cost_per_test": 0.00484, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 93.2, "latency": 9.123, "stderr": 1.592, "cost_per_test": 0.00667, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 93.2, "latency": 13.831, "stderr": 1.592, "cost_per_test": 0.000533, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 92.8, "latency": 2.454, "stderr": 1.635, "cost_per_test": 0.00019, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 92.8, "latency": 3.485, "stderr": 1.635, "cost_per_test": 0.004426, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 92.8, "latency": 5.139, "stderr": 1.635, "cost_per_test": 0.000516, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3p2": { "accuracy": 92.8, "latency": 9.57, "stderr": 1.635, "cost_per_test": 0.000161, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 92.8, "latency": 11.95, "stderr": 1.635, "cost_per_test": 0.012954, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3-0324": { "accuracy": 92.8, "latency": 15.944, "stderr": 1.635, "cost_per_test": 0.000361, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "zai/glm-4.5": { "accuracy": 92.8, "latency": 121.536, "stderr": 1.635, "cost_per_test": 0.009316, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-2-1212": { "accuracy": 92.4, "latency": 7.354, "stderr": 1.676, "cost_per_test": 0.004538, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 92.4, "latency": 7.901, "stderr": 1.676, "cost_per_test": 0.003138, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-large-2512": { "accuracy": 92.4, "latency": 8.05, "stderr": 1.676, "cost_per_test": 0.000532, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 92.4, "latency": 8.829, "stderr": 1.676, "cost_per_test": 0.000855, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "minimax/MiniMax-M2.1": { "accuracy": 92.4, "latency": 13.369, "stderr": 1.676, "cost_per_test": 0.001451, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "zai/glm-4.6": { "accuracy": 92.4, "latency": 37.073, "stderr": 1.676, "cost_per_test": 0.005126, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 92.0, "latency": 4.495, "stderr": 1.716, "cost_per_test": 0.005119, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3": { "accuracy": 92.0, "latency": 11.034, "stderr": 1.716, "cost_per_test": 0.000341, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 91.6, "latency": 1.375, "stderr": 1.754, "cost_per_test": 0.000191, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 91.6, "latency": 2.243, "stderr": 1.754, "cost_per_test": 0.00036, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-medium-2505": { "accuracy": 91.6, "latency": 5.599, "stderr": 1.754, "cost_per_test": 0.00063, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 91.2, "latency": 4.493, "stderr": 1.792, "cost_per_test": 0.000164, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 91.2, "latency": 11.849, "stderr": 1.792, "cost_per_test": 0.001144, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "kimi/kimi-k2-thinking": { "accuracy": 91.2, "latency": 68.233, "stderr": 1.792, "cost_per_test": 0.001208, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-1.5-pro-002": { "accuracy": 90.8, "latency": 2.635, "stderr": 1.828, "cost_per_test": 0.00098, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 90.4, "latency": 1.518, "stderr": 1.863, "cost_per_test": 0.000328, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/gpt-oss-20b": { "accuracy": 90.4, "latency": 7.04, "stderr": 1.863, "cost_per_test": 0.000541, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 90.4, "latency": 21.998, "stderr": 1.863, "cost_per_test": 0.002, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 89.2, "latency": 3.532, "stderr": 1.963, "cost_per_test": 0.000145, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-flash-002": { "accuracy": 88.0, "latency": 1.289, "stderr": 2.055, "cost_per_test": 0.000058, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 87.2, "latency": 29.317, "stderr": 2.113, "cost_per_test": 0.000411, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "cohere/command-a-03-2025": { "accuracy": 86.4, "latency": 7.094, "stderr": 2.168, "cost_per_test": 0.004219, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "mistralai/mistral-small-2402": { "accuracy": 86.0, "latency": 3.389, "stderr": 2.194, "cost_per_test": 0.000249, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-large-2411": { "accuracy": 86.0, "latency": 9.515, "stderr": 2.194, "cost_per_test": 0.003496, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 85.2, "latency": 3.264, "stderr": 2.246, "cost_per_test": 0.000173, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 84.8, "latency": 5.057, "stderr": 2.271, "cost_per_test": 0.00148, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-small-2503": { "accuracy": 81.2, "latency": 4.998, "stderr": 2.471, "cost_per_test": 0.00013, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/magistral-small-2509": { "accuracy": 80.0, "latency": 8.767, "stderr": 2.53, "cost_per_test": 0.001679, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2509": { "accuracy": 72.0, "latency": 10.409, "stderr": 2.84, "cost_per_test": 0.005653, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "ai21labs/jamba-large-1.6": { "accuracy": 70.4, "latency": 10.788, "stderr": 2.887, "cost_per_test": 0.004038, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 54.8, "latency": 1.386, "stderr": 3.148, "cost_per_test": 0.000122, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 23.2, "latency": 4.176, "stderr": 2.67, "cost_per_test": 0.000312, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_es": { "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 97.2, "latency": 10.935, "stderr": 1.043, "cost_per_test": 0.057625, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 96.8, "latency": 8.484, "stderr": 1.113, "cost_per_test": 0.008955, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 96.4, "latency": 4.92, "stderr": 1.178, "cost_per_test": 0.004392, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 96.4, "latency": 6.788, "stderr": 1.178, "cost_per_test": 0.021058, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o4-mini-2025-04-16": { "accuracy": 96.4, "latency": 6.847, "stderr": 1.178, "cost_per_test": 0.002462, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-pro-preview": { "accuracy": 96.4, "latency": 11.73, "stderr": 1.178, "cost_per_test": 0.016573, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3-flash-preview": { "accuracy": 96.4, "latency": 12.371, "stderr": 1.178, "cost_per_test": 0.006273, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 96.0, "latency": 3.035, "stderr": 1.239, "cost_per_test": 0.003214, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 96.0, "latency": 8.333, "stderr": 1.239, "cost_per_test": 0.021065, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 96.0, "latency": 12.116, "stderr": 1.239, "cost_per_test": 0.048402, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3": { "accuracy": 96.0, "latency": 16.554, "stderr": 1.239, "cost_per_test": 0.000319, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 95.6, "latency": 3.549, "stderr": 1.297, "cost_per_test": 0.004267, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3-0324": { "accuracy": 95.6, "latency": 10.991, "stderr": 1.297, "cost_per_test": 0.000324, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/gpt-oss-120b": { "accuracy": 95.2, "latency": 4.707, "stderr": 1.352, "cost_per_test": 0.000552, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 95.2, "latency": 9.548, "stderr": 1.352, "cost_per_test": 0.000781, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 95.2, "latency": 16.329, "stderr": 1.352, "cost_per_test": 0.019816, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/qwen3-235b-a22b": { "accuracy": 95.2, "latency": 29.593, "stderr": 1.352, "cost_per_test": 0.00097, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-3": { "accuracy": 94.8, "latency": 3.67, "stderr": 1.404, "cost_per_test": 0.004265, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-r1": { "accuracy": 94.8, "latency": 8.761, "stderr": 1.404, "cost_per_test": 0.003549, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 94.8, "latency": 12.098, "stderr": 1.404, "cost_per_test": 0.003339, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max-preview": { "accuracy": 94.8, "latency": 16.488, "stderr": 1.404, "cost_per_test": 0.001549, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "mistralai/mistral-medium-2505": { "accuracy": 94.4, "latency": 3.579, "stderr": 1.454, "cost_per_test": 0.000524, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 94.4, "latency": 11.992, "stderr": 1.454, "cost_per_test": 0.010437, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 94.0, "latency": 1.69, "stderr": 1.502, "cost_per_test": 0.000304, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-large-2512": { "accuracy": 94.0, "latency": 9.102, "stderr": 1.502, "cost_per_test": 0.00041, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 93.6, "latency": 2.337, "stderr": 1.548, "cost_per_test": 0.000186, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "fireworks/deepseek-v3p2": { "accuracy": 93.6, "latency": 8.459, "stderr": 1.548, "cost_per_test": 0.000142, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 93.6, "latency": 12.228, "stderr": 1.548, "cost_per_test": 0.012277, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "kimi/kimi-k2-thinking": { "accuracy": 93.6, "latency": 31.332, "stderr": 1.548, "cost_per_test": 0.000598, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 93.2, "latency": 1.297, "stderr": 1.592, "cost_per_test": 0.000171, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 93.2, "latency": 2.331, "stderr": 1.592, "cost_per_test": 0.000337, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-2-1212": { "accuracy": 93.2, "latency": 4.291, "stderr": 1.592, "cost_per_test": 0.002415, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max": { "accuracy": 93.2, "latency": 7.531, "stderr": 1.592, "cost_per_test": 0.001868, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 93.2, "latency": 21.076, "stderr": 1.592, "cost_per_test": 0.001929, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-0709": { "accuracy": 93.2, "latency": 157.382, "stderr": 1.592, "cost_per_test": 0.022483, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.0-flash-001": { "accuracy": 92.8, "latency": 1.707, "stderr": 1.635, "cost_per_test": 0.000109, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 92.8, "latency": 4.724, "stderr": 1.635, "cost_per_test": 0.02208, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/magistral-small-2509": { "accuracy": 92.8, "latency": 4.753, "stderr": 1.635, "cost_per_test": 0.001096, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "cohere/command-a-03-2025": { "accuracy": 92.8, "latency": 11.81, "stderr": 1.635, "cost_per_test": 0.002455, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "google/gemini-1.5-pro-002": { "accuracy": 92.4, "latency": 2.238, "stderr": 1.676, "cost_per_test": 0.00082, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 92.4, "latency": 7.851, "stderr": 1.676, "cost_per_test": 0.000639, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/magistral-medium-2509": { "accuracy": 92.4, "latency": 8.714, "stderr": 1.676, "cost_per_test": 0.00459, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "zai/glm-4.5": { "accuracy": 92.4, "latency": 54.816, "stderr": 1.676, "cost_per_test": 0.003995, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 92.0, "latency": 1.783, "stderr": 1.716, "cost_per_test": 0.00028, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 92.0, "latency": 2.974, "stderr": 1.716, "cost_per_test": 0.000482, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 92.0, "latency": 7.568, "stderr": 1.716, "cost_per_test": 0.005545, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 92.0, "latency": 11.849, "stderr": 1.716, "cost_per_test": 0.051907, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.1": { "accuracy": 92.0, "latency": 12.475, "stderr": 1.716, "cost_per_test": 0.001105, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "fireworks/gpt-oss-20b": { "accuracy": 91.6, "latency": 4.547, "stderr": 1.754, "cost_per_test": 0.000359, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-2025-04-16": { "accuracy": 91.6, "latency": 5.745, "stderr": 1.754, "cost_per_test": 0.00322, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 91.6, "latency": 8.999, "stderr": 1.754, "cost_per_test": 0.005862, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 91.2, "latency": 2.791, "stderr": 1.792, "cost_per_test": 0.000169, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 91.2, "latency": 3.157, "stderr": 1.792, "cost_per_test": 0.000119, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/mistral-small-2503": { "accuracy": 91.2, "latency": 3.555, "stderr": 1.792, "cost_per_test": 0.00008, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "zai/glm-4.6": { "accuracy": 91.2, "latency": 27.061, "stderr": 1.792, "cost_per_test": 0.003385, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "zai/glm-4.7": { "accuracy": 91.2, "latency": 42.764, "stderr": 1.792, "cost_per_test": 0.003068, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 90.8, "latency": 4.206, "stderr": 1.828, "cost_per_test": 0.000417, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 90.8, "latency": 25.681, "stderr": 1.828, "cost_per_test": 0.016827, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-11-20": { "accuracy": 90.4, "latency": 3.661, "stderr": 1.863, "cost_per_test": 0.003856, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 90.4, "latency": 4.064, "stderr": 1.863, "cost_per_test": 0.00012, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/o3-mini-2025-01-31": { "accuracy": 90.4, "latency": 16.116, "stderr": 1.863, "cost_per_test": 0.009147, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 90.0, "latency": 3.282, "stderr": 1.897, "cost_per_test": 0.000833, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 90.0, "latency": 5.736, "stderr": 1.897, "cost_per_test": 0.000846, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 89.6, "latency": 1.77, "stderr": 1.931, "cost_per_test": 0.001346, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-small-2402": { "accuracy": 89.6, "latency": 2.477, "stderr": 1.931, "cost_per_test": 0.000169, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-large-2411": { "accuracy": 89.6, "latency": 6.594, "stderr": 1.931, "cost_per_test": 0.002169, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 89.6, "latency": 6.717, "stderr": 1.931, "cost_per_test": 0.002973, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 89.2, "latency": 2.626, "stderr": 1.963, "cost_per_test": 0.000111, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 89.2, "latency": 3.181, "stderr": 1.963, "cost_per_test": 0.000983, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 86.4, "latency": 1.12, "stderr": 2.168, "cost_per_test": 0.000097, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-1.5-flash-002": { "accuracy": 86.4, "latency": 1.127, "stderr": 2.168, "cost_per_test": 0.000048, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 86.0, "latency": 9.599, "stderr": 2.194, "cost_per_test": 0.00039, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "ai21labs/jamba-large-1.6": { "accuracy": 83.6, "latency": 6.249, "stderr": 2.342, "cost_per_test": 0.002106, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 64.8, "latency": 21.698, "stderr": 3.021, "cost_per_test": 0.000304, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 62.8, "latency": 2.282, "stderr": 3.057, "cost_per_test": 0.000134, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_te": { "google/gemini-3-pro-preview": { "accuracy": 95.6, "latency": 15.56, "stderr": 1.297, "cost_per_test": 0.022621, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 94.8, "latency": 17.401, "stderr": 1.404, "cost_per_test": 0.067623, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 94.0, "latency": 10.511, "stderr": 1.502, "cost_per_test": 0.005416, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 94.0, "latency": 10.957, "stderr": 1.502, "cost_per_test": 0.04183, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 93.6, "latency": 11.722, "stderr": 1.548, "cost_per_test": 0.013355, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 93.2, "latency": 8.194, "stderr": 1.592, "cost_per_test": 0.042174, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 92.8, "latency": 16.575, "stderr": 1.635, "cost_per_test": 0.091721, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 92.0, "latency": 11.989, "stderr": 1.716, "cost_per_test": 0.041574, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 92.0, "latency": 15.762, "stderr": 1.716, "cost_per_test": 0.01775, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 91.6, "latency": 33.3, "stderr": 1.754, "cost_per_test": 0.039673, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/qwen3-235b-a22b": { "accuracy": 90.8, "latency": 51.592, "stderr": 1.828, "cost_per_test": 0.001592, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 90.4, "latency": 7.16, "stderr": 1.863, "cost_per_test": 0.001607, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 90.0, "latency": 4.104, "stderr": 1.897, "cost_per_test": 0.000352, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 90.0, "latency": 7.298, "stderr": 1.897, "cost_per_test": 0.009122, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 89.6, "latency": 11.426, "stderr": 1.931, "cost_per_test": 0.009245, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o1-2024-12-17": { "accuracy": 89.6, "latency": 12.716, "stderr": 1.931, "cost_per_test": 0.07567, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 89.6, "latency": 18.668, "stderr": 1.931, "cost_per_test": 0.005868, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 89.6, "latency": 24.386, "stderr": 1.931, "cost_per_test": 0.017629, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 89.6, "latency": 28.159, "stderr": 1.931, "cost_per_test": 0.031337, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 89.6, "latency": 51.493, "stderr": 1.931, "cost_per_test": 0.000766, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 89.2, "latency": 8.864, "stderr": 1.963, "cost_per_test": 0.009551, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2": { "accuracy": 89.2, "latency": 13.042, "stderr": 1.963, "cost_per_test": 0.000245, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "fireworks/deepseek-r1": { "accuracy": 89.2, "latency": 16.308, "stderr": 1.963, "cost_per_test": 0.006924, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 89.2, "latency": 18.792, "stderr": 1.963, "cost_per_test": 0.013626, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4-fast-reasoning": { "accuracy": 88.8, "latency": 6.529, "stderr": 1.995, "cost_per_test": 0.001155, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-120b": { "accuracy": 88.8, "latency": 8.109, "stderr": 1.995, "cost_per_test": 0.000936, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 88.4, "latency": 3.35, "stderr": 2.025, "cost_per_test": 0.002975, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 88.4, "latency": 4.293, "stderr": 2.025, "cost_per_test": 0.001637, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "zai/glm-4.6": { "accuracy": 88.4, "latency": 67.945, "stderr": 2.025, "cost_per_test": 0.008821, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 88.0, "latency": 5.397, "stderr": 2.055, "cost_per_test": 0.000253, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/o4-mini-2025-04-16": { "accuracy": 88.0, "latency": 10.461, "stderr": 2.055, "cost_per_test": 0.003704, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-large-2512": { "accuracy": 88.0, "latency": 11.17, "stderr": 2.055, "cost_per_test": 0.000616, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-3": { "accuracy": 88.0, "latency": 11.171, "stderr": 2.055, "cost_per_test": 0.01059, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-preview": { "accuracy": 88.0, "latency": 48.433, "stderr": 2.055, "cost_per_test": 0.006589, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "google/gemini-2.0-flash-001": { "accuracy": 87.6, "latency": 1.788, "stderr": 2.084, "cost_per_test": 0.000142, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/o3-mini-2025-01-31": { "accuracy": 87.6, "latency": 18.362, "stderr": 2.084, "cost_per_test": 0.011167, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max": { "accuracy": 87.6, "latency": 27.879, "stderr": 2.084, "cost_per_test": 0.00753, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-4o-2024-11-20": { "accuracy": 87.2, "latency": 4.12, "stderr": 2.113, "cost_per_test": 0.005378, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 87.2, "latency": 6.051, "stderr": 2.113, "cost_per_test": 0.006176, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 87.2, "latency": 23.31, "stderr": 2.113, "cost_per_test": 0.001628, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3": { "accuracy": 87.2, "latency": 28.802, "stderr": 2.113, "cost_per_test": 0.00076, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 86.8, "latency": 7.961, "stderr": 2.141, "cost_per_test": 0.003276, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-medium-2505": { "accuracy": 86.8, "latency": 10.947, "stderr": 2.141, "cost_per_test": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 86.8, "latency": 16.27, "stderr": 2.141, "cost_per_test": 0.001541, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 86.8, "latency": 17.17, "stderr": 2.141, "cost_per_test": 0.001032, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-pro-002": { "accuracy": 86.4, "latency": 3.954, "stderr": 2.168, "cost_per_test": 0.001663, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/gpt-oss-20b": { "accuracy": 86.4, "latency": 8.033, "stderr": 2.168, "cost_per_test": 0.00064, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-2025-04-16": { "accuracy": 86.4, "latency": 8.142, "stderr": 2.168, "cost_per_test": 0.004823, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.7": { "accuracy": 86.4, "latency": 98.258, "stderr": 2.168, "cost_per_test": 0.009035, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "zai/glm-4.5": { "accuracy": 86.4, "latency": 321.186, "stderr": 2.168, "cost_per_test": 0.015304, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-0709": { "accuracy": 86.0, "latency": 206.881, "stderr": 2.194, "cost_per_test": 0.043618, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3-0324": { "accuracy": 85.6, "latency": 18.414, "stderr": 2.22, "cost_per_test": 0.00075, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "kimi/kimi-k2-thinking": { "accuracy": 85.6, "latency": 91.07, "stderr": 2.22, "cost_per_test": 0.001542, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 85.2, "latency": 2.29, "stderr": 2.246, "cost_per_test": 0.0006, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 85.2, "latency": 27.775, "stderr": 2.246, "cost_per_test": 0.001001, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 84.8, "latency": 1.986, "stderr": 2.271, "cost_per_test": 0.00038, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 84.8, "latency": 4.856, "stderr": 2.271, "cost_per_test": 0.00146, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 84.4, "latency": 6.731, "stderr": 2.295, "cost_per_test": 0.000261, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "minimax/MiniMax-M2.1": { "accuracy": 84.4, "latency": 13.656, "stderr": 2.295, "cost_per_test": 0.001356, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "grok/grok-2-1212": { "accuracy": 83.6, "latency": 18.976, "stderr": 2.342, "cost_per_test": 0.011334, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-flash-002": { "accuracy": 83.2, "latency": 2.49, "stderr": 2.364, "cost_per_test": 0.000121, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 82.4, "latency": 3.564, "stderr": 2.408, "cost_per_test": 0.000764, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-large-2411": { "accuracy": 80.8, "latency": 18.094, "stderr": 2.491, "cost_per_test": 0.007035, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 80.8, "latency": 27.569, "stderr": 2.491, "cost_per_test": 0.002256, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 78.4, "latency": 4.903, "stderr": 2.603, "cost_per_test": 0.000216, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/magistral-small-2509": { "accuracy": 78.0, "latency": 17.77, "stderr": 2.62, "cost_per_test": 0.002109, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 76.4, "latency": 5.178, "stderr": 2.686, "cost_per_test": 0.000225, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 75.2, "latency": 5.201, "stderr": 2.731, "cost_per_test": 0.001523, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-small-2503": { "accuracy": 74.4, "latency": 5.083, "stderr": 2.76, "cost_per_test": 0.000138, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2402": { "accuracy": 72.0, "latency": 5.3, "stderr": 2.84, "cost_per_test": 0.000368, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2509": { "accuracy": 63.6, "latency": 20.244, "stderr": 3.043, "cost_per_test": 0.007005, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 59.6, "latency": 2.723, "stderr": 3.103, "cost_per_test": 0.000294, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "cohere/command-a-03-2025": { "accuracy": 56.4, "latency": 14.898, "stderr": 3.136, "cost_per_test": 0.011344, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "ai21labs/jamba-large-1.6": { "accuracy": 36.8, "latency": 28.633, "stderr": 3.05, "cost_per_test": 0.011834, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "ai21labs/jamba-mini-1.6": { "accuracy": 13.6, "latency": 10.775, "stderr": 2.168, "cost_per_test": 0.00084, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "mgsm_sw": { "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 97.2, "latency": 11.593, "stderr": 1.043, "cost_per_test": 0.0536, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 97.2, "latency": 13.239, "stderr": 1.043, "cost_per_test": 0.018586, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3-flash-preview": { "accuracy": 97.2, "latency": 15.125, "stderr": 1.043, "cost_per_test": 0.007734, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 96.4, "latency": 6.511, "stderr": 1.178, "cost_per_test": 0.02487, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 96.4, "latency": 10.802, "stderr": 1.178, "cost_per_test": 0.011713, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 95.6, "latency": 7.332, "stderr": 1.297, "cost_per_test": 0.005489, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o3-2025-04-16": { "accuracy": 95.6, "latency": 9.31, "stderr": 1.297, "cost_per_test": 0.003818, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 95.2, "latency": 3.821, "stderr": 1.352, "cost_per_test": 0.000873, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 95.2, "latency": 7.437, "stderr": 1.352, "cost_per_test": 0.022936, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 95.2, "latency": 13.675, "stderr": 1.352, "cost_per_test": 0.049456, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 94.8, "latency": 10.859, "stderr": 1.404, "cost_per_test": 0.009554, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 94.8, "latency": 40.354, "stderr": 1.404, "cost_per_test": 0.026773, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 94.4, "latency": 23.264, "stderr": 1.454, "cost_per_test": 0.015183, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 94.0, "latency": 6.627, "stderr": 1.502, "cost_per_test": 0.000868, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 94.0, "latency": 7.344, "stderr": 1.502, "cost_per_test": 0.005051, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 93.2, "latency": 6.536, "stderr": 1.592, "cost_per_test": 0.003379, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 93.2, "latency": 9.356, "stderr": 1.592, "cost_per_test": 0.055666, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 93.2, "latency": 14.238, "stderr": 1.592, "cost_per_test": 0.0055, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 93.2, "latency": 17.427, "stderr": 1.592, "cost_per_test": 0.00066, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 93.2, "latency": 17.466, "stderr": 1.592, "cost_per_test": 0.014858, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 93.2, "latency": 19.486, "stderr": 1.592, "cost_per_test": 0.022279, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 92.8, "latency": 2.853, "stderr": 1.635, "cost_per_test": 0.001402, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 92.8, "latency": 33.9, "stderr": 1.635, "cost_per_test": 0.000481, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 92.4, "latency": 6.638, "stderr": 1.676, "cost_per_test": 0.004637, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 92.0, "latency": 2.886, "stderr": 1.716, "cost_per_test": 0.000255, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 92.0, "latency": 16.42, "stderr": 1.716, "cost_per_test": 0.000797, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4o-2024-11-20": { "accuracy": 91.6, "latency": 3.156, "stderr": 1.754, "cost_per_test": 0.00373, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 91.6, "latency": 3.291, "stderr": 1.754, "cost_per_test": 0.002293, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3": { "accuracy": 91.6, "latency": 5.026, "stderr": 1.754, "cost_per_test": 0.004118, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 91.6, "latency": 12.991, "stderr": 1.754, "cost_per_test": 0.007978, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 91.6, "latency": 17.507, "stderr": 1.754, "cost_per_test": 0.00541, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2-thinking": { "accuracy": 91.6, "latency": 60.627, "stderr": 1.754, "cost_per_test": 0.001036, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "zai/glm-4.6": { "accuracy": 91.2, "latency": 33.856, "stderr": 1.792, "cost_per_test": 0.004308, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 90.8, "latency": 5.003, "stderr": 1.828, "cost_per_test": 0.000727, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 90.8, "latency": 5.292, "stderr": 1.828, "cost_per_test": 0.000148, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 90.8, "latency": 5.377, "stderr": 1.828, "cost_per_test": 0.000505, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-r1": { "accuracy": 90.8, "latency": 10.572, "stderr": 1.828, "cost_per_test": 0.004247, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.0-flash-001": { "accuracy": 90.4, "latency": 1.483, "stderr": 1.863, "cost_per_test": 0.000098, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/qwen3-235b-a22b": { "accuracy": 90.4, "latency": 25.275, "stderr": 1.863, "cost_per_test": 0.001306, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 90.0, "latency": 1.548, "stderr": 1.897, "cost_per_test": 0.000189, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3p2": { "accuracy": 90.0, "latency": 10.087, "stderr": 1.897, "cost_per_test": 0.000161, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 89.6, "latency": 2.692, "stderr": 1.931, "cost_per_test": 0.000182, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "alibaba/qwen3-max": { "accuracy": 89.6, "latency": 10.787, "stderr": 1.931, "cost_per_test": 0.002274, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "zai/glm-4.7": { "accuracy": 89.6, "latency": 76.56, "stderr": 1.931, "cost_per_test": 0.004452, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 89.2, "latency": 3.51, "stderr": 1.963, "cost_per_test": 0.00041, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "openai/o3-mini-2025-01-31": { "accuracy": 89.2, "latency": 44.064, "stderr": 1.963, "cost_per_test": 0.010974, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.5": { "accuracy": 89.2, "latency": 73.632, "stderr": 1.963, "cost_per_test": 0.005937, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "fireworks/gpt-oss-120b": { "accuracy": 88.8, "latency": 6.781, "stderr": 1.995, "cost_per_test": 0.000803, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3-max-preview": { "accuracy": 88.8, "latency": 20.87, "stderr": 1.995, "cost_per_test": 0.001731, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "fireworks/deepseek-v3-0324": { "accuracy": 88.4, "latency": 4.678, "stderr": 2.025, "cost_per_test": 0.000422, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/deepseek-v3": { "accuracy": 88.4, "latency": 6.294, "stderr": 2.025, "cost_per_test": 0.000389, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-1.5-pro-002": { "accuracy": 88.0, "latency": 2.513, "stderr": 2.055, "cost_per_test": 0.000881, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 88.0, "latency": 10.187, "stderr": 2.055, "cost_per_test": 0.000903, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "minimax/MiniMax-M2.1": { "accuracy": 88.0, "latency": 11.344, "stderr": 2.055, "cost_per_test": 0.001074, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "mistralai/mistral-large-2512": { "accuracy": 87.6, "latency": 7.226, "stderr": 2.084, "cost_per_test": 0.000407, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 86.8, "latency": 29.052, "stderr": 2.141, "cost_per_test": 0.001971, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 86.4, "latency": 10.703, "stderr": 2.168, "cost_per_test": 0.000347, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 86.0, "latency": 3.814, "stderr": 2.194, "cost_per_test": 0.000156, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-1.5-flash-002": { "accuracy": 85.6, "latency": 1.208, "stderr": 2.22, "cost_per_test": 0.000056, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-0709": { "accuracy": 85.2, "latency": 77.338, "stderr": 2.246, "cost_per_test": 0.035632, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-2-1212": { "accuracy": 84.8, "latency": 1.957, "stderr": 2.271, "cost_per_test": 0.002114, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 84.8, "latency": 3.751, "stderr": 2.271, "cost_per_test": 0.000134, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 84.0, "latency": 1.626, "stderr": 2.319, "cost_per_test": 0.000321, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-medium-2505": { "accuracy": 83.2, "latency": 3.452, "stderr": 2.364, "cost_per_test": 0.000536, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2509": { "accuracy": 82.4, "latency": 13.49, "stderr": 2.408, "cost_per_test": 0.006031, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/magistral-small-2509": { "accuracy": 78.0, "latency": 13.885, "stderr": 2.62, "cost_per_test": 0.001576, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-large-2411": { "accuracy": 77.2, "latency": 5.32, "stderr": 2.653, "cost_per_test": 0.002487, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/gpt-oss-20b": { "accuracy": 76.4, "latency": 9.154, "stderr": 2.686, "cost_per_test": 0.000748, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "cohere/command-a-03-2025": { "accuracy": 74.4, "latency": 3.983, "stderr": 2.76, "cost_per_test": 0.002741, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "mistralai/mistral-small-2402": { "accuracy": 73.6, "latency": 1.707, "stderr": 2.788, "cost_per_test": 0.000173, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 72.0, "latency": 2.442, "stderr": 2.84, "cost_per_test": 0.00009, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 71.6, "latency": 3.613, "stderr": 2.852, "cost_per_test": 0.001045, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 64.4, "latency": 1.937, "stderr": 3.028, "cost_per_test": 0.000114, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "ai21labs/jamba-large-1.6": { "accuracy": 34.0, "latency": 5.402, "stderr": 2.996, "cost_per_test": 0.001391, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "ai21labs/jamba-mini-1.6": { "accuracy": 12.0, "latency": 2.442, "stderr": 2.055, "cost_per_test": 0.000164, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } } } }