{ "metadata": { "benchmark": "Finance Agent v1", "slug": "finance_agent_v1", "description": "Evaluating agents on core financial analyst tasks", "benchmark_id": "finance_agent_v1", "updated": "2026-02-10", "dataset_type": "private", "industry": "finance", "tasks": { "overall": "Overall", "simple_retrieval_quantitative": "Simple retrieval - Quantitative", "simple_retrieval_qualitative": "Simple retrieval - Qualitative", "complex_retrieval": "Complex Retrieval", "numerical_reasoning": "Numerical Reasoning", "financial_modeling": "Financial Modeling Projections", "market_analysis": "Market Analysis", "beat_or_miss": "Beat or Miss", "trends": "Trends", "adjustments": "Adjustments" }, "total_models": 59, "models": [ "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3-max-preview", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-20250514-thinking", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-sonnet-4-5-20250929-thinking", "cohere/command-a-03-2025", "fireworks/gpt-oss-120b", "fireworks/llama4-maverick-instruct-basic", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-preview-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "minimax/MiniMax-M2.1", "mistralai/magistral-medium-2506", "mistralai/magistral-medium-2509", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-small-2503", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "zai/glm-4.6", "zai/glm-4.7" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "family": "finance_agent", "version": "1", "archived": true, "runner": "custom", "mode": "agentic" }, "tasks": { "overall": { "anthropic/claude-opus-4-6-thinking": { "accuracy": 60.6537, "latency": 251.1514, "stderr": 1.072, "cost_per_test": 1.460254, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 56.5528, "latency": 356.627, "stderr": 1.139, "cost_per_test": 0.44323, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 55.3232, "latency": 140.8643, "stderr": 1.1496, "cost_per_test": 0.981899, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 55.232, "latency": 126.9531, "stderr": 1.1439, "cost_per_test": 1.167628, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 54.5167, "latency": 84.8069, "stderr": 1.137, "cost_per_test": 1.189665, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 52.7948, "latency": 358.7118, "stderr": 1.178, "cost_per_test": 0.939474, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 50.8892, "latency": 141.2137, "stderr": 1.1971, "cost_per_test": 3.802345, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 49.2495, "latency": 111.3328, "stderr": 1.1933, "cost_per_test": 1.149563, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "kimi/kimi-k2.5-thinking": { "accuracy": 49.2399, "latency": 245.5813, "stderr": 1.2199, "cost_per_test": 0.141314, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "openai/o3-2025-04-16": { "accuracy": 47.6944, "latency": 45.1963, "stderr": 1.2212, "cost_per_test": 0.130181, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 46.8901, "latency": 458.528, "stderr": 1.2192, "cost_per_test": 0.690356, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 46.0645, "latency": 118.947, "stderr": 1.2199, "cost_per_test": 3.73444, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 44.5262, "latency": 45.1963, "stderr": 1.2192, "cost_per_test": 0.198131, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 44.1354, "latency": 121.9327, "stderr": 1.2245, "cost_per_test": 0.52729, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 44.0773, "latency": 45.1963, "stderr": 1.232, "cost_per_test": 0.198131, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 44.0105, "latency": 192.8798, "stderr": 1.2301, "cost_per_test": 0.055425, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 43.6056, "latency": 135.4814, "stderr": 1.2172, "cost_per_test": 3.705654, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 43.4912, "latency": 94.1698, "stderr": 1.2338, "cost_per_test": 0.857452, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 43.2294, "latency": 471.4992, "stderr": 1.243, "cost_per_test": 1.277493, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 43.0426, "latency": 98.2393, "stderr": 1.2295, "cost_per_test": 0.282041, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 42.872, "latency": 45.1963, "stderr": 1.2276, "cost_per_test": 0.198131, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 42.346, "latency": 100.7996, "stderr": 1.2307, "cost_per_test": 3.843008, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "zai/glm-4.7": { "accuracy": 41.8748, "latency": 386.0473, "stderr": 1.2505, "cost_per_test": 0.201365, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "grok/grok-4-0709": { "accuracy": 40.2615, "latency": 446.4825, "stderr": 1.2468, "cost_per_test": 1.10794, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-3-flash-preview": { "accuracy": 39.3239, "latency": 892.6527, "stderr": 1.2805, "cost_per_test": 0.357442, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 37.8518, "latency": 196.3802, "stderr": 1.2644, "cost_per_test": 0.088984, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 37.0629, "latency": 61.7117, "stderr": 1.2584, "cost_per_test": 0.20043, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 35.879, "latency": 45.1963, "stderr": 1.2889, "cost_per_test": 0.0716, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "alibaba/qwen3-max": { "accuracy": 32.1744, "latency": 222.1956, "stderr": 1.302, "cost_per_test": 0.329085, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 31.6753, "latency": 45.1963, "stderr": 1.3035, "cost_per_test": 0.01886, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 31.1632, "latency": 47.3274, "stderr": 1.3012, "cost_per_test": 0.063933, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 30.6538, "latency": 66.6855, "stderr": 1.3023, "cost_per_test": 0.075946, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "minimax/MiniMax-M2.1": { "accuracy": 29.7193, "latency": 145.8614, "stderr": 1.3017, "cost_per_test": 0.106596, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 29.3657, "latency": 45.1963, "stderr": 0.0, "cost_per_test": 0.086129, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "fireworks/gpt-oss-120b": { "accuracy": 28.3164, "latency": 188.3849, "stderr": 1.3026, "cost_per_test": 0.071511, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "zai/glm-4.6": { "accuracy": 27.0627, "latency": 132.5813, "stderr": 1.3024, "cost_per_test": 0.241952, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "kimi/kimi-k2-thinking": { "accuracy": 25.9945, "latency": 125.0259, "stderr": 1.3028, "cost_per_test": 0.095624, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 24.7567, "latency": 36.7626, "stderr": 1.3013, "cost_per_test": 0.105802, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 24.5723, "latency": 45.1963, "stderr": 1.3026, "cost_per_test": 0.130181, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-3": { "accuracy": 24.0764, "latency": 45.1963, "stderr": 1.3027, "cost_per_test": 0.198131, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2512": { "accuracy": 23.5155, "latency": 220.0312, "stderr": 1.2782, "cost_per_test": 0.109253, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 20.837, "latency": 45.1963, "stderr": 1.2833, "cost_per_test": 0.026036, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/o1-2024-12-17": { "accuracy": 20.7635, "latency": 45.1963, "stderr": 1.288, "cost_per_test": 0.976357, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 19.3336, "latency": 45.1963, "stderr": 1.2742, "cost_per_test": 0.162726, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 18.2258, "latency": 352.2806, "stderr": 1.2458, "cost_per_test": 0.484308, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "alibaba/qwen3-max-preview": { "accuracy": 17.7401, "latency": 178.7293, "stderr": 1.2711, "cost_per_test": 0.378365, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 17.0917, "latency": 45.1963, "stderr": 1.2629, "cost_per_test": 0.01886, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 14.3353, "latency": 45.1963, "stderr": 1.2362, "cost_per_test": 0.052835, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-2.0-flash-001": { "accuracy": 13.1666, "latency": 45.1963, "stderr": 1.2029, "cost_per_test": 0.006509, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/o3-mini-2025-01-31": { "accuracy": 12.7166, "latency": 45.1963, "stderr": 1.1899, "cost_per_test": 0.0716, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 10.7692, "latency": 45.1963, "stderr": 0.0, "cost_per_test": 0.009764, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-small-2503": { "accuracy": 10.002, "latency": 45.1963, "stderr": 1.1915, "cost_per_test": 0.004882, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-medium-2505": { "accuracy": 8.8479, "latency": 45.1963, "stderr": 1.1712, "cost_per_test": 0.026417, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2506": { "accuracy": 7.4553, "latency": 45.1963, "stderr": 1.1133, "cost_per_test": 0.127321, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "cohere/command-a-03-2025": { "accuracy": 4.2887, "latency": 45.1963, "stderr": 1.1574, "cost_per_test": 0.162726, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 3.635, "latency": 45.1963, "stderr": 0.9569, "cost_per_test": 0.01432, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 3.3588, "latency": 45.1963, "stderr": 0.9299, "cost_per_test": 0.054763, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 2.4739, "latency": 45.1963, "stderr": 1.072, "cost_per_test": 0.006509, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 1.7417, "latency": 45.1963, "stderr": 1.0441, "cost_per_test": 0.012637, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" } }, "simple_retrieval_quantitative": { "anthropic/claude-opus-4-6-thinking": { "accuracy": 78.125, "latency": 151.731, "stderr": 5.1675, "cost_per_test": 1.053315, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 75.0, "latency": 44.1029, "stderr": 5.4127, "cost_per_test": 0.199196, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 75.0, "latency": 79.5188, "stderr": 5.4127, "cost_per_test": 0.858865, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 73.4375, "latency": 44.1029, "stderr": 5.5208, "cost_per_test": 0.199196, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 73.4375, "latency": 66.386, "stderr": 5.5208, "cost_per_test": 0.652396, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 73.4375, "latency": 96.2476, "stderr": 5.5208, "cost_per_test": 0.454254, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "kimi/kimi-k2.5-thinking": { "accuracy": 71.875, "latency": 138.5159, "stderr": 5.6201, "cost_per_test": 0.101796, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 71.875, "latency": 212.321, "stderr": 5.6201, "cost_per_test": 0.250055, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 71.875, "latency": 306.7421, "stderr": 5.6201, "cost_per_test": 0.760982, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 70.3125, "latency": 49.5348, "stderr": 5.711, "cost_per_test": 0.995753, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 70.3125, "latency": 90.1869, "stderr": 5.711, "cost_per_test": 0.600493, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 70.3125, "latency": 97.7763, "stderr": 5.711, "cost_per_test": 3.090534, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 68.75, "latency": 44.1029, "stderr": 5.7939, "cost_per_test": 0.199196, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 68.75, "latency": 64.2826, "stderr": 5.7939, "cost_per_test": 0.169369, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 68.75, "latency": 67.1782, "stderr": 5.7939, "cost_per_test": 3.004259, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 68.75, "latency": 88.248, "stderr": 5.7939, "cost_per_test": 0.973451, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 67.1875, "latency": 88.1121, "stderr": 5.8691, "cost_per_test": 3.015309, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 67.1875, "latency": 94.9883, "stderr": 5.8691, "cost_per_test": 2.807618, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 67.1875, "latency": 140.844, "stderr": 5.8691, "cost_per_test": 0.046211, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 67.1875, "latency": 346.3271, "stderr": 5.8691, "cost_per_test": 0.730609, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "openai/gpt-5-2025-08-07": { "accuracy": 65.625, "latency": 316.2824, "stderr": 5.937, "cost_per_test": 0.46159, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/o3-2025-04-16": { "accuracy": 64.0625, "latency": 44.1029, "stderr": 5.9977, "cost_per_test": 0.131097, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "zai/glm-4.7": { "accuracy": 62.5, "latency": 255.8747, "stderr": 6.0515, "cost_per_test": 0.139358, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 57.8125, "latency": 141.9621, "stderr": 6.1732, "cost_per_test": 0.061217, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max": { "accuracy": 57.8125, "latency": 149.7342, "stderr": 6.1732, "cost_per_test": 0.218202, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 56.25, "latency": 32.8627, "stderr": 6.201, "cost_per_test": 0.046813, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "zai/glm-4.6": { "accuracy": 56.25, "latency": 105.7453, "stderr": 6.201, "cost_per_test": 0.202001, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 54.6875, "latency": 40.2739, "stderr": 6.2225, "cost_per_test": 0.14422, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-4-0709": { "accuracy": 54.6875, "latency": 373.4291, "stderr": 6.2225, "cost_per_test": 0.887081, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 53.125, "latency": 44.1029, "stderr": 6.2378, "cost_per_test": 0.01907, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 53.125, "latency": 44.1029, "stderr": 6.2378, "cost_per_test": 0.072104, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.1": { "accuracy": 53.125, "latency": 106.9208, "stderr": 6.2378, "cost_per_test": 0.076477, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "google/gemini-3-flash-preview": { "accuracy": 51.5625, "latency": 641.0852, "stderr": 6.2469, "cost_per_test": 0.250917, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "fireworks/gpt-oss-120b": { "accuracy": 50.0, "latency": 108.6251, "stderr": 6.25, "cost_per_test": 0.051913, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 48.4375, "latency": 44.1029, "stderr": 6.2469, "cost_per_test": 0.131097, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 48.4375, "latency": 52.0667, "stderr": 6.2469, "cost_per_test": 0.052146, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/gpt-4o-2024-08-06": { "accuracy": 46.875, "latency": 44.1029, "stderr": 6.2378, "cost_per_test": 0.163872, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 45.3125, "latency": 44.1029, "stderr": 6.2225, "cost_per_test": 0.086186, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/o1-2024-12-17": { "accuracy": 45.3125, "latency": 44.1029, "stderr": 6.2225, "cost_per_test": 0.98323, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 45.3125, "latency": 190.8267, "stderr": 6.2225, "cost_per_test": 0.366488, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 43.75, "latency": 24.5929, "stderr": 6.201, "cost_per_test": 0.08055, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "kimi/kimi-k2-thinking": { "accuracy": 43.75, "latency": 97.3967, "stderr": 6.201, "cost_per_test": 0.065914, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "grok/grok-3": { "accuracy": 40.625, "latency": 44.1029, "stderr": 6.1392, "cost_per_test": 0.199196, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.0-flash-001": { "accuracy": 39.0625, "latency": 44.1029, "stderr": 6.0986, "cost_per_test": 0.006555, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "alibaba/qwen3-max-preview": { "accuracy": 39.0625, "latency": 75.5604, "stderr": 6.0986, "cost_per_test": 0.186962, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 37.5, "latency": 44.1029, "stderr": 6.0515, "cost_per_test": 0.009832, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 37.5, "latency": 44.1029, "stderr": 6.0515, "cost_per_test": 0.026219, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 35.9375, "latency": 44.1029, "stderr": 5.9977, "cost_per_test": 0.01907, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 34.375, "latency": 44.1029, "stderr": 5.937, "cost_per_test": 0.053119, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/o3-mini-2025-01-31": { "accuracy": 34.375, "latency": 44.1029, "stderr": 5.937, "cost_per_test": 0.072104, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-large-2512": { "accuracy": 32.8125, "latency": 132.2802, "stderr": 5.8691, "cost_per_test": 0.065256, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 25.0, "latency": 44.1029, "stderr": 5.4127, "cost_per_test": 0.004916, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-medium-2505": { "accuracy": 20.3125, "latency": 44.1029, "stderr": 5.0291, "cost_per_test": 0.026559, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2506": { "accuracy": 15.625, "latency": 44.1029, "stderr": 4.5387, "cost_per_test": 0.128547, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "cohere/command-a-03-2025": { "accuracy": 15.625, "latency": 44.1029, "stderr": 4.5387, "cost_per_test": 0.163872, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 7.8125, "latency": 44.1029, "stderr": 3.3546, "cost_per_test": 0.014421, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 3.125, "latency": 44.1029, "stderr": 2.1749, "cost_per_test": 0.055439, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 1.5625, "latency": 44.1029, "stderr": 1.5502, "cost_per_test": 0.006555, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.0, "latency": 44.1029, "stderr": 0.0, "cost_per_test": 0.01277, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" } }, "simple_retrieval_qualitative": { "anthropic/claude-opus-4-6-thinking": { "accuracy": 77.0492, "latency": 138.5859, "stderr": 5.3842, "cost_per_test": 0.875562, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 73.7705, "latency": 78.1444, "stderr": 5.6321, "cost_per_test": 0.748714, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 73.7705, "latency": 94.3271, "stderr": 5.6321, "cost_per_test": 0.949327, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 72.1311, "latency": 61.6549, "stderr": 5.7406, "cost_per_test": 0.686553, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 70.4918, "latency": 246.2655, "stderr": 5.8395, "cost_per_test": 0.310986, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 67.2131, "latency": 87.8934, "stderr": 6.0105, "cost_per_test": 0.613938, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 67.2131, "latency": 113.8711, "stderr": 6.0105, "cost_per_test": 2.913909, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "kimi/kimi-k2.5-thinking": { "accuracy": 67.2131, "latency": 143.3565, "stderr": 6.0105, "cost_per_test": 0.081095, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 62.2951, "latency": 37.4088, "stderr": 6.2053, "cost_per_test": 0.173042, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 62.2951, "latency": 229.0658, "stderr": 6.2053, "cost_per_test": 0.518973, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 62.2951, "latency": 281.4374, "stderr": 6.2053, "cost_per_test": 0.702572, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 60.6557, "latency": 37.4088, "stderr": 6.2548, "cost_per_test": 0.173042, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 60.6557, "latency": 94.1831, "stderr": 6.2548, "cost_per_test": 2.793502, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 60.6557, "latency": 204.3714, "stderr": 6.2548, "cost_per_test": 0.057253, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "zai/glm-4.7": { "accuracy": 60.6557, "latency": 294.783, "stderr": 6.2548, "cost_per_test": 0.140075, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "openai/o3-2025-04-16": { "accuracy": 59.0164, "latency": 37.4088, "stderr": 6.2969, "cost_per_test": 0.113643, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 59.0164, "latency": 37.4088, "stderr": 6.2969, "cost_per_test": 0.173042, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 57.377, "latency": 64.0078, "stderr": 6.3318, "cost_per_test": 0.154918, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 57.377, "latency": 83.9867, "stderr": 6.3318, "cost_per_test": 0.344881, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "anthropic/claude-opus-4-20250514": { "accuracy": 57.377, "latency": 87.7807, "stderr": 6.3318, "cost_per_test": 2.858314, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 57.377, "latency": 146.8475, "stderr": 6.3318, "cost_per_test": 0.034424, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 55.7377, "latency": 112.5169, "stderr": 6.3596, "cost_per_test": 2.666748, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 54.0984, "latency": 69.9595, "stderr": 6.3803, "cost_per_test": 0.523099, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "minimax/MiniMax-M2.1": { "accuracy": 52.459, "latency": 99.5913, "stderr": 6.3941, "cost_per_test": 0.051966, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "openai/gpt-5-2025-08-07": { "accuracy": 50.8197, "latency": 406.6561, "stderr": 6.401, "cost_per_test": 0.643532, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "kimi/kimi-k2-thinking": { "accuracy": 47.541, "latency": 76.4023, "stderr": 6.3941, "cost_per_test": 0.056048, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "fireworks/gpt-oss-120b": { "accuracy": 45.9016, "latency": 96.2659, "stderr": 6.3803, "cost_per_test": 0.040103, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "grok/grok-4-0709": { "accuracy": 45.9016, "latency": 301.321, "stderr": 6.3803, "cost_per_test": 0.58195, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 44.2623, "latency": 37.4088, "stderr": 6.3596, "cost_per_test": 0.022729, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 44.2623, "latency": 37.4088, "stderr": 6.3596, "cost_per_test": 0.075324, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "grok/grok-4-fast-reasoning": { "accuracy": 44.2623, "latency": 40.9672, "stderr": 6.3596, "cost_per_test": 0.099012, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-3-flash-preview": { "accuracy": 44.2623, "latency": 808.6136, "stderr": 6.3596, "cost_per_test": 0.271739, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 42.623, "latency": 24.5433, "stderr": 6.3318, "cost_per_test": 0.060981, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 42.623, "latency": 37.4088, "stderr": 6.3318, "cost_per_test": 0.016445, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max": { "accuracy": 42.623, "latency": 126.5402, "stderr": 6.3318, "cost_per_test": 0.178925, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "zai/glm-4.6": { "accuracy": 40.9836, "latency": 101.6287, "stderr": 6.2969, "cost_per_test": 0.15353, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "openai/o4-mini-2025-04-16": { "accuracy": 39.3443, "latency": 37.4088, "stderr": 6.2548, "cost_per_test": 0.062503, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 37.7049, "latency": 37.4088, "stderr": 6.2053, "cost_per_test": 0.016445, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 36.0656, "latency": 37.4088, "stderr": 6.1482, "cost_per_test": 0.113643, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 36.0656, "latency": 37.4088, "stderr": 6.1482, "cost_per_test": 0.142053, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 34.4262, "latency": 36.2362, "stderr": 6.0834, "cost_per_test": 0.045771, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "mistralai/mistral-large-2512": { "accuracy": 34.4262, "latency": 145.0258, "stderr": 6.0834, "cost_per_test": 0.075305, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "grok/grok-3": { "accuracy": 32.7869, "latency": 37.4088, "stderr": 6.0105, "cost_per_test": 0.173042, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-preview": { "accuracy": 32.7869, "latency": 84.1064, "stderr": 6.0105, "cost_per_test": 0.12026, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 29.5082, "latency": 47.7567, "stderr": 5.8395, "cost_per_test": 0.047905, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 27.8689, "latency": 37.4088, "stderr": 5.7406, "cost_per_test": 0.046145, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 27.8689, "latency": 37.4088, "stderr": 5.7406, "cost_per_test": 0.852319, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 24.5902, "latency": 37.4088, "stderr": 5.5135, "cost_per_test": 0.008523, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.0-flash-001": { "accuracy": 22.9508, "latency": 37.4088, "stderr": 5.3842, "cost_per_test": 0.005682, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/o3-mini-2025-01-31": { "accuracy": 22.9508, "latency": 37.4088, "stderr": 5.3842, "cost_per_test": 0.062503, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 21.3115, "latency": 224.4644, "stderr": 5.2432, "cost_per_test": 0.226702, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-medium-2505": { "accuracy": 18.0328, "latency": 37.4088, "stderr": 4.9225, "cost_per_test": 0.023072, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 14.7541, "latency": 37.4088, "stderr": 4.5408, "cost_per_test": 0.004262, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2506": { "accuracy": 11.4754, "latency": 37.4088, "stderr": 4.0809, "cost_per_test": 0.111064, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "cohere/command-a-03-2025": { "accuracy": 9.8361, "latency": 37.4088, "stderr": 3.813, "cost_per_test": 0.142053, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" }, "ai21labs/jamba-mini-1.6": { "accuracy": 8.1967, "latency": 37.4088, "stderr": 3.5122, "cost_per_test": 0.01102, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 8.1967, "latency": 37.4088, "stderr": 3.5122, "cost_per_test": 0.047734, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 6.5574, "latency": 37.4088, "stderr": 3.1694, "cost_per_test": 0.005682, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 3.2787, "latency": 37.4088, "stderr": 2.2801, "cost_per_test": 0.012501, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" } }, "complex_retrieval": { "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 66.6667, "latency": 186.027, "stderr": 11.1111, "cost_per_test": 1.208596, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 61.1111, "latency": 104.4653, "stderr": 11.4904, "cost_per_test": 1.445046, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 61.1111, "latency": 200.6056, "stderr": 11.4904, "cost_per_test": 5.641491, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 61.1111, "latency": 390.0032, "stderr": 11.4904, "cost_per_test": 0.660988, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 55.5556, "latency": 47.5281, "stderr": 11.7121, "cost_per_test": 0.167978, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-fast-reasoning": { "accuracy": 50.0, "latency": 67.7687, "stderr": 11.7851, "cost_per_test": 0.215127, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 50.0, "latency": 150.3189, "stderr": 11.7851, "cost_per_test": 1.25262, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-0709": { "accuracy": 50.0, "latency": 239.2806, "stderr": 11.7851, "cost_per_test": 1.127223, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "kimi/kimi-k2.5-thinking": { "accuracy": 50.0, "latency": 342.2063, "stderr": 11.7851, "cost_per_test": 0.206668, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 50.0, "latency": 353.024, "stderr": 11.7851, "cost_per_test": 1.872775, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 50.0, "latency": 493.2561, "stderr": 11.7851, "cost_per_test": 1.285048, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 47.3684, "latency": 174.9248, "stderr": 11.4549, "cost_per_test": 0.082148, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 44.4444, "latency": 47.4351, "stderr": 11.7121, "cost_per_test": 0.069012, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/o3-2025-04-16": { "accuracy": 44.4444, "latency": 47.5281, "stderr": 11.7121, "cost_per_test": 0.109956, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 44.4444, "latency": 142.4582, "stderr": 11.7121, "cost_per_test": 0.435508, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 44.4444, "latency": 155.3782, "stderr": 11.7121, "cost_per_test": 5.042624, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 44.4444, "latency": 599.916, "stderr": 11.7121, "cost_per_test": 0.95341, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 38.8889, "latency": 47.5281, "stderr": 11.4904, "cost_per_test": 0.167978, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 38.8889, "latency": 79.8819, "stderr": 11.4904, "cost_per_test": 0.086643, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 38.8889, "latency": 100.8599, "stderr": 11.4904, "cost_per_test": 1.124898, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 38.8889, "latency": 132.184, "stderr": 11.4904, "cost_per_test": 0.637068, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 38.8889, "latency": 159.2861, "stderr": 11.4904, "cost_per_test": 4.453377, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 38.8889, "latency": 162.7085, "stderr": 11.4904, "cost_per_test": 0.172952, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "mistralai/magistral-medium-2509": { "accuracy": 38.8889, "latency": 333.9927, "stderr": 11.4904, "cost_per_test": 0.543773, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "google/gemini-3-flash-preview": { "accuracy": 38.8889, "latency": 418.6632, "stderr": 11.4904, "cost_per_test": 0.289841, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 38.8889, "latency": 577.65, "stderr": 11.4904, "cost_per_test": 1.83053, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 33.3333, "latency": 37.8955, "stderr": 11.1111, "cost_per_test": 0.101888, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 33.3333, "latency": 47.5281, "stderr": 11.1111, "cost_per_test": 0.060476, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 33.3333, "latency": 47.5281, "stderr": 11.1111, "cost_per_test": 0.073796, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 33.3333, "latency": 47.5281, "stderr": 11.1111, "cost_per_test": 0.167978, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 33.3333, "latency": 107.7241, "stderr": 11.1111, "cost_per_test": 1.217983, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 33.3333, "latency": 121.6999, "stderr": 11.1111, "cost_per_test": 4.733444, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "minimax/MiniMax-M2.1": { "accuracy": 33.3333, "latency": 157.6224, "stderr": 11.1111, "cost_per_test": 0.095083, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 27.7778, "latency": 47.5281, "stderr": 10.5572, "cost_per_test": 0.015783, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "openai/o1-2024-12-17": { "accuracy": 27.7778, "latency": 47.5281, "stderr": 10.5572, "cost_per_test": 0.824672, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "zai/glm-4.6": { "accuracy": 27.7778, "latency": 178.8132, "stderr": 10.5572, "cost_per_test": 0.345436, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "zai/glm-4.7": { "accuracy": 27.7778, "latency": 823.8402, "stderr": 10.5572, "cost_per_test": 0.492632, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 22.2222, "latency": 47.5281, "stderr": 9.7991, "cost_per_test": 0.109956, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 22.2222, "latency": 47.5281, "stderr": 9.7991, "cost_per_test": 0.137445, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-large-2512": { "accuracy": 22.2222, "latency": 197.8269, "stderr": 9.7991, "cost_per_test": 0.108517, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "alibaba/qwen3-max": { "accuracy": 22.2222, "latency": 260.3713, "stderr": 9.7991, "cost_per_test": 0.391978, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "fireworks/gpt-oss-120b": { "accuracy": 22.2222, "latency": 333.7433, "stderr": 9.7991, "cost_per_test": 0.118791, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "mistralai/mistral-small-2503": { "accuracy": 16.6667, "latency": 47.5281, "stderr": 8.7841, "cost_per_test": 0.004123, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-medium-2505": { "accuracy": 16.6667, "latency": 47.5281, "stderr": 8.7841, "cost_per_test": 0.022397, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 16.6667, "latency": 47.5281, "stderr": 8.7841, "cost_per_test": 0.044794, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/o3-mini-2025-01-31": { "accuracy": 16.6667, "latency": 47.5281, "stderr": 8.7841, "cost_per_test": 0.060476, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-3": { "accuracy": 16.6667, "latency": 47.5281, "stderr": 8.7841, "cost_per_test": 0.167978, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "kimi/kimi-k2-thinking": { "accuracy": 16.6667, "latency": 169.9366, "stderr": 8.7841, "cost_per_test": 0.144908, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 11.1111, "latency": 47.5281, "stderr": 7.4074, "cost_per_test": 0.008247, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 11.1111, "latency": 47.5281, "stderr": 7.4074, "cost_per_test": 0.012095, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 11.1111, "latency": 47.5281, "stderr": 7.4074, "cost_per_test": 0.015783, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "mistralai/magistral-medium-2506": { "accuracy": 11.1111, "latency": 47.5281, "stderr": 7.4074, "cost_per_test": 0.106912, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "alibaba/qwen3-max-preview": { "accuracy": 11.1111, "latency": 126.3815, "stderr": 7.4074, "cost_per_test": 0.325722, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "google/gemini-2.0-flash-001": { "accuracy": 5.5556, "latency": 47.5281, "stderr": 5.399, "cost_per_test": 0.005498, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 5.5556, "latency": 47.5281, "stderr": 5.399, "cost_per_test": 0.005498, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 5.5556, "latency": 47.5281, "stderr": 5.399, "cost_per_test": 0.01059, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 5.5556, "latency": 47.5281, "stderr": 5.399, "cost_per_test": 0.021991, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 5.5556, "latency": 47.5281, "stderr": 5.399, "cost_per_test": 0.045702, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "cohere/command-a-03-2025": { "accuracy": 5.5556, "latency": 47.5281, "stderr": 5.399, "cost_per_test": 0.137445, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" } }, "numerical_reasoning": { "openai/gpt-5.2-2025-12-11": { "accuracy": 69.2308, "latency": 268.7454, "stderr": 6.4004, "cost_per_test": 0.703219, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "openai/o3-2025-04-16": { "accuracy": 63.4615, "latency": 49.1788, "stderr": 6.6777, "cost_per_test": 0.152094, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 63.4615, "latency": 206.4828, "stderr": 6.6777, "cost_per_test": 1.492185, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 61.5385, "latency": 68.5475, "stderr": 6.7466, "cost_per_test": 1.231684, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 59.6154, "latency": 278.1471, "stderr": 6.8043, "cost_per_test": 0.447058, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 55.7692, "latency": 49.1788, "stderr": 6.8874, "cost_per_test": 0.231075, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 55.7692, "latency": 106.0749, "stderr": 6.8874, "cost_per_test": 1.245108, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 55.7692, "latency": 114.0246, "stderr": 6.8874, "cost_per_test": 0.896216, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 55.7692, "latency": 228.9357, "stderr": 6.8874, "cost_per_test": 0.054967, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-3-pro-preview": { "accuracy": 53.8462, "latency": 115.4507, "stderr": 6.9132, "cost_per_test": 0.522567, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "kimi/kimi-k2.5-thinking": { "accuracy": 53.8462, "latency": 281.1195, "stderr": 6.9132, "cost_per_test": 0.178094, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 51.9231, "latency": 81.932, "stderr": 6.9286, "cost_per_test": 0.963597, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 51.9231, "latency": 82.7171, "stderr": 6.9286, "cost_per_test": 0.293655, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 50.0, "latency": 88.4367, "stderr": 6.9338, "cost_per_test": 3.701976, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 50.0, "latency": 120.7672, "stderr": 6.9338, "cost_per_test": 3.574546, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 50.0, "latency": 134.8531, "stderr": 6.9338, "cost_per_test": 3.796122, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 50.0, "latency": 384.9829, "stderr": 6.9338, "cost_per_test": 0.576031, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-0709": { "accuracy": 50.0, "latency": 429.8998, "stderr": 6.9338, "cost_per_test": 1.283607, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 48.0769, "latency": 454.7078, "stderr": 6.9286, "cost_per_test": 0.950953, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "google/gemini-3-flash-preview": { "accuracy": 48.0769, "latency": 798.2887, "stderr": 6.9286, "cost_per_test": 0.301442, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 46.1538, "latency": 49.1788, "stderr": 6.9132, "cost_per_test": 0.231075, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 46.1538, "latency": 107.4922, "stderr": 6.9132, "cost_per_test": 3.530357, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/o4-mini-2025-04-16": { "accuracy": 44.2308, "latency": 49.1788, "stderr": 6.8874, "cost_per_test": 0.083652, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 44.2308, "latency": 197.5652, "stderr": 6.8874, "cost_per_test": 0.102657, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "zai/glm-4.7": { "accuracy": 44.2308, "latency": 364.9149, "stderr": 6.8874, "cost_per_test": 0.190238, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 42.3077, "latency": 49.1788, "stderr": 6.8512, "cost_per_test": 0.099949, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 42.3077, "latency": 49.1788, "stderr": 6.8512, "cost_per_test": 0.231075, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-fast-reasoning": { "accuracy": 42.3077, "latency": 65.7441, "stderr": 6.8512, "cost_per_test": 0.227859, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 40.3846, "latency": 71.2803, "stderr": 6.8043, "cost_per_test": 0.075644, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 40.3846, "latency": 82.9734, "stderr": 6.8043, "cost_per_test": 0.862779, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 32.6923, "latency": 49.1788, "stderr": 6.5051, "cost_per_test": 0.02213, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "openai/o1-2024-12-17": { "accuracy": 32.6923, "latency": 49.1788, "stderr": 6.5051, "cost_per_test": 1.140707, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "zai/glm-4.6": { "accuracy": 32.6923, "latency": 113.4705, "stderr": 6.5051, "cost_per_test": 0.217338, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 30.7692, "latency": 55.0178, "stderr": 6.4004, "cost_per_test": 0.064713, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "kimi/kimi-k2-thinking": { "accuracy": 30.7692, "latency": 127.3212, "stderr": 6.4004, "cost_per_test": 0.107457, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "mistralai/mistral-large-2512": { "accuracy": 30.7692, "latency": 151.9576, "stderr": 6.4004, "cost_per_test": 0.105897, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "alibaba/qwen3-max": { "accuracy": 28.8462, "latency": 212.6055, "stderr": 6.2826, "cost_per_test": 0.34065, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "minimax/MiniMax-M2.1": { "accuracy": 26.9231, "latency": 134.8242, "stderr": 6.1511, "cost_per_test": 0.103476, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "fireworks/gpt-oss-120b": { "accuracy": 25.0, "latency": 176.2747, "stderr": 6.0048, "cost_per_test": 0.069663, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 23.0769, "latency": 49.1788, "stderr": 5.8427, "cost_per_test": 0.030419, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 23.0769, "latency": 49.1788, "stderr": 5.8427, "cost_per_test": 0.152094, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 21.1538, "latency": 33.8281, "stderr": 5.6635, "cost_per_test": 0.104288, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 19.2308, "latency": 49.1788, "stderr": 5.4654, "cost_per_test": 0.02213, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-3": { "accuracy": 19.2308, "latency": 49.1788, "stderr": 5.4654, "cost_per_test": 0.231075, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 17.3077, "latency": 49.1788, "stderr": 5.2463, "cost_per_test": 0.06162, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-4o-2024-08-06": { "accuracy": 17.3077, "latency": 49.1788, "stderr": 5.2463, "cost_per_test": 0.190118, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 17.3077, "latency": 412.135, "stderr": 5.2463, "cost_per_test": 0.561223, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "alibaba/qwen3-max-preview": { "accuracy": 15.3846, "latency": 205.7741, "stderr": 5.0034, "cost_per_test": 0.403561, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "google/gemini-2.0-flash-001": { "accuracy": 9.6154, "latency": 49.1788, "stderr": 4.0882, "cost_per_test": 0.007605, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/o3-mini-2025-01-31": { "accuracy": 9.6154, "latency": 49.1788, "stderr": 4.0882, "cost_per_test": 0.083652, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 7.6923, "latency": 49.1788, "stderr": 3.6953, "cost_per_test": 0.011407, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-small-2503": { "accuracy": 5.7692, "latency": 49.1788, "stderr": 3.2334, "cost_per_test": 0.005704, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 3.8462, "latency": 49.1788, "stderr": 2.6668, "cost_per_test": 0.01673, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 1.9231, "latency": 49.1788, "stderr": 1.9045, "cost_per_test": 0.007605, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 1.9231, "latency": 49.1788, "stderr": 1.9045, "cost_per_test": 0.014818, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" }, "mistralai/mistral-medium-2505": { "accuracy": 1.9231, "latency": 49.1788, "stderr": 1.9045, "cost_per_test": 0.03081, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 1.9231, "latency": 49.1788, "stderr": 1.9045, "cost_per_test": 0.064339, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "cohere/command-a-03-2025": { "accuracy": 1.9231, "latency": 49.1788, "stderr": 1.9045, "cost_per_test": 0.190118, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" }, "mistralai/magistral-medium-2506": { "accuracy": 0.0, "latency": 49.1788, "stderr": 0.0, "cost_per_test": 0.14916, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" } }, "financial_modeling": { "openai/gpt-5.2-2025-12-11": { "accuracy": 73.3333, "latency": 277.7881, "stderr": 8.0737, "cost_per_test": 0.619597, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 73.3333, "latency": 340.8901, "stderr": 8.0737, "cost_per_test": 0.327732, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 70.0, "latency": 94.9634, "stderr": 8.3666, "cost_per_test": 0.924577, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 70.0, "latency": 262.2883, "stderr": 8.3666, "cost_per_test": 1.187212, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 70.0, "latency": 311.4479, "stderr": 8.3666, "cost_per_test": 0.372444, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/o3-2025-04-16": { "accuracy": 66.6667, "latency": 37.1346, "stderr": 8.6066, "cost_per_test": 0.117741, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 66.6667, "latency": 56.175, "stderr": 8.6066, "cost_per_test": 0.810428, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 66.6667, "latency": 112.9236, "stderr": 8.6066, "cost_per_test": 0.720338, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 63.3333, "latency": 83.6707, "stderr": 8.7981, "cost_per_test": 0.779434, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 63.3333, "latency": 131.8198, "stderr": 8.7981, "cost_per_test": 0.445399, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 60.0, "latency": 109.0335, "stderr": 8.9443, "cost_per_test": 3.569828, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 60.0, "latency": 121.2736, "stderr": 8.9443, "cost_per_test": 3.011951, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 56.6667, "latency": 83.1759, "stderr": 9.0472, "cost_per_test": 3.127874, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 56.6667, "latency": 118.7437, "stderr": 9.0472, "cost_per_test": 3.053225, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 56.6667, "latency": 182.65, "stderr": 9.0472, "cost_per_test": 0.039598, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "kimi/kimi-k2.5-thinking": { "accuracy": 56.6667, "latency": 197.8648, "stderr": 9.0472, "cost_per_test": 0.107259, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "google/gemini-3-flash-preview": { "accuracy": 56.6667, "latency": 1819.8178, "stderr": 9.0472, "cost_per_test": 0.506302, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 53.3333, "latency": 37.1346, "stderr": 9.1084, "cost_per_test": 0.017049, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 53.3333, "latency": 37.1346, "stderr": 9.1084, "cost_per_test": 0.064757, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 53.3333, "latency": 37.1346, "stderr": 9.1084, "cost_per_test": 0.179233, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-fast-reasoning": { "accuracy": 53.3333, "latency": 42.7991, "stderr": 9.1084, "cost_per_test": 0.102834, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 53.3333, "latency": 74.0159, "stderr": 9.1084, "cost_per_test": 0.565395, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 50.0, "latency": 67.0251, "stderr": 9.1287, "cost_per_test": 0.214929, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "zai/glm-4.7": { "accuracy": 50.0, "latency": 255.3102, "stderr": 9.1287, "cost_per_test": 0.12928, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 50.0, "latency": 397.5631, "stderr": 9.1287, "cost_per_test": 0.654286, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "grok/grok-3": { "accuracy": 46.6667, "latency": 37.1346, "stderr": 9.1084, "cost_per_test": 0.179233, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 46.6667, "latency": 37.1346, "stderr": 9.1084, "cost_per_test": 0.179233, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-0709": { "accuracy": 46.6667, "latency": 567.1529, "stderr": 9.1084, "cost_per_test": 1.193929, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 43.3333, "latency": 37.1346, "stderr": 9.0472, "cost_per_test": 0.179233, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "minimax/MiniMax-M2.1": { "accuracy": 43.3333, "latency": 128.4564, "stderr": 9.0472, "cost_per_test": 0.08626, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 40.0, "latency": 38.4757, "stderr": 8.9443, "cost_per_test": 0.034042, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "kimi/kimi-k2-thinking": { "accuracy": 40.0, "latency": 123.4197, "stderr": 8.9443, "cost_per_test": 0.091733, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "mistralai/mistral-large-2512": { "accuracy": 40.0, "latency": 147.5604, "stderr": 8.9443, "cost_per_test": 0.079273, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "alibaba/qwen3-max": { "accuracy": 40.0, "latency": 162.9637, "stderr": 8.9443, "cost_per_test": 0.216562, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "fireworks/gpt-oss-120b": { "accuracy": 40.0, "latency": 210.5173, "stderr": 8.9443, "cost_per_test": 0.045784, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 36.6667, "latency": 37.1346, "stderr": 8.7981, "cost_per_test": 0.077958, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 36.6667, "latency": 37.1346, "stderr": 8.7981, "cost_per_test": 0.117741, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 36.6667, "latency": 45.9772, "stderr": 8.7981, "cost_per_test": 0.036241, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "zai/glm-4.6": { "accuracy": 36.6667, "latency": 96.432, "stderr": 8.7981, "cost_per_test": 0.149497, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 36.6667, "latency": 191.8457, "stderr": 8.7981, "cost_per_test": 0.06774, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 33.3333, "latency": 37.1346, "stderr": 8.6066, "cost_per_test": 0.023548, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 33.3333, "latency": 37.1346, "stderr": 8.6066, "cost_per_test": 0.147176, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 30.0, "latency": 31.2185, "stderr": 8.3666, "cost_per_test": 0.092376, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-preview": { "accuracy": 30.0, "latency": 128.2364, "stderr": 8.3666, "cost_per_test": 0.26256, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "openai/o1-2024-12-17": { "accuracy": 26.6667, "latency": 37.1346, "stderr": 8.0737, "cost_per_test": 0.883055, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 23.3333, "latency": 251.1208, "stderr": 7.722, "cost_per_test": 0.368844, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-medium-2505": { "accuracy": 16.6667, "latency": 37.1346, "stderr": 6.8041, "cost_per_test": 0.023898, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "openai/o3-mini-2025-01-31": { "accuracy": 16.6667, "latency": 37.1346, "stderr": 6.8041, "cost_per_test": 0.064757, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-small-2503": { "accuracy": 13.3333, "latency": 37.1346, "stderr": 6.2063, "cost_per_test": 0.004415, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 13.3333, "latency": 37.1346, "stderr": 6.2063, "cost_per_test": 0.017049, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 13.3333, "latency": 37.1346, "stderr": 6.2063, "cost_per_test": 0.047796, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "mistralai/magistral-medium-2506": { "accuracy": 13.3333, "latency": 37.1346, "stderr": 6.2063, "cost_per_test": 0.115118, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "google/gemini-2.0-flash-001": { "accuracy": 10.0, "latency": 37.1346, "stderr": 5.4772, "cost_per_test": 0.005887, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 10.0, "latency": 37.1346, "stderr": 5.4772, "cost_per_test": 0.008831, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 6.6667, "latency": 37.1346, "stderr": 4.5542, "cost_per_test": 0.005887, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 6.6667, "latency": 37.1346, "stderr": 4.5542, "cost_per_test": 0.012951, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 6.6667, "latency": 37.1346, "stderr": 4.5542, "cost_per_test": 0.049498, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "cohere/command-a-03-2025": { "accuracy": 3.3333, "latency": 37.1346, "stderr": 3.2773, "cost_per_test": 0.147176, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.0, "latency": 37.1346, "stderr": 0.0, "cost_per_test": 0.011424, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" } }, "market_analysis": { "anthropic/claude-opus-4-6-thinking": { "accuracy": 42.8571, "latency": 571.4729, "stderr": 10.799, "cost_per_test": 3.027857, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 33.3333, "latency": 244.183, "stderr": 10.2869, "cost_per_test": 2.24069, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 33.3333, "latency": 287.5511, "stderr": 10.2869, "cost_per_test": 2.548834, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 33.3333, "latency": 335.4254, "stderr": 10.2869, "cost_per_test": 2.944401, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 28.5714, "latency": 776.9074, "stderr": 9.8581, "cost_per_test": 0.879633, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 28.5714, "latency": 777.6388, "stderr": 9.8581, "cost_per_test": 1.911094, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 28.5714, "latency": 810.1794, "stderr": 9.8581, "cost_per_test": 1.440564, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/o3-2025-04-16": { "accuracy": 23.8095, "latency": 47.0132, "stderr": 9.2943, "cost_per_test": 0.151012, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 23.8095, "latency": 170.1606, "stderr": 9.2943, "cost_per_test": 0.616802, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 23.8095, "latency": 191.6482, "stderr": 9.2943, "cost_per_test": 2.401058, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 23.8095, "latency": 249.4379, "stderr": 9.2943, "cost_per_test": 6.244951, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "kimi/kimi-k2.5-thinking": { "accuracy": 23.8095, "latency": 368.8438, "stderr": 9.2943, "cost_per_test": 0.217478, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "grok/grok-4-0709": { "accuracy": 23.8095, "latency": 808.6161, "stderr": 9.2943, "cost_per_test": 1.93374, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-3-pro-preview": { "accuracy": 19.0476, "latency": 166.1332, "stderr": 8.5689, "cost_per_test": 1.08801, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "zai/glm-4.7": { "accuracy": 19.0476, "latency": 484.6664, "stderr": 8.5689, "cost_per_test": 0.288441, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 14.2857, "latency": 47.0132, "stderr": 7.636, "cost_per_test": 0.230232, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 14.2857, "latency": 173.5173, "stderr": 7.636, "cost_per_test": 1.59849, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 14.2857, "latency": 236.8627, "stderr": 7.636, "cost_per_test": 7.22117, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 14.2857, "latency": 249.8716, "stderr": 7.636, "cost_per_test": 6.328159, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 14.2857, "latency": 251.4836, "stderr": 7.636, "cost_per_test": 0.103913, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 14.2857, "latency": 778.2528, "stderr": 7.636, "cost_per_test": 2.96301, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "google/gemini-3-flash-preview": { "accuracy": 14.2857, "latency": 819.069, "stderr": 7.636, "cost_per_test": 0.630192, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 13.6364, "latency": 207.7437, "stderr": 7.3165, "cost_per_test": 0.089463, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 9.5238, "latency": 47.0132, "stderr": 6.4056, "cost_per_test": 0.021785, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 9.5238, "latency": 47.0132, "stderr": 6.4056, "cost_per_test": 0.030202, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 9.5238, "latency": 47.0132, "stderr": 6.4056, "cost_per_test": 0.083057, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 9.5238, "latency": 47.0132, "stderr": 6.4056, "cost_per_test": 0.151012, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 9.5238, "latency": 47.0132, "stderr": 6.4056, "cost_per_test": 0.230232, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-3": { "accuracy": 9.5238, "latency": 47.0132, "stderr": 6.4056, "cost_per_test": 0.230232, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 9.5238, "latency": 65.735, "stderr": 6.4056, "cost_per_test": 0.13022, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "grok/grok-4-fast-reasoning": { "accuracy": 9.5238, "latency": 97.9449, "stderr": 6.4056, "cost_per_test": 0.305535, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-20250514": { "accuracy": 9.5238, "latency": 187.8699, "stderr": 6.4056, "cost_per_test": 7.324234, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "alibaba/qwen3-max-preview": { "accuracy": 9.5238, "latency": 205.3401, "stderr": 6.4056, "cost_per_test": 0.856766, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "fireworks/gpt-oss-120b": { "accuracy": 9.5238, "latency": 342.2143, "stderr": 6.4056, "cost_per_test": 0.11756, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "alibaba/qwen3-max": { "accuracy": 9.0909, "latency": 325.5872, "stderr": 6.1291, "cost_per_test": 0.453637, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "google/gemini-2.0-flash-001": { "accuracy": 4.7619, "latency": 47.0132, "stderr": 4.6471, "cost_per_test": 0.007551, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 4.7619, "latency": 47.0132, "stderr": 4.6471, "cost_per_test": 0.021785, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 4.7619, "latency": 47.0132, "stderr": 4.6471, "cost_per_test": 0.063177, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 4.7619, "latency": 47.0132, "stderr": 4.6471, "cost_per_test": 0.100573, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/gpt-4o-2024-08-06": { "accuracy": 4.7619, "latency": 47.0132, "stderr": 4.6471, "cost_per_test": 0.188765, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 4.7619, "latency": 47.0132, "stderr": 4.6471, "cost_per_test": 0.230232, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 4.7619, "latency": 47.0132, "stderr": 4.6471, "cost_per_test": 1.13259, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 4.7619, "latency": 53.2872, "stderr": 4.6471, "cost_per_test": 0.120623, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 4.7619, "latency": 122.9637, "stderr": 4.6471, "cost_per_test": 0.178734, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "kimi/kimi-k2-thinking": { "accuracy": 4.7619, "latency": 174.558, "stderr": 4.6471, "cost_per_test": 0.155039, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "zai/glm-4.6": { "accuracy": 4.7619, "latency": 200.7384, "stderr": 4.6471, "cost_per_test": 0.412682, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "minimax/MiniMax-M2.1": { "accuracy": 4.7619, "latency": 214.4455, "stderr": 4.6471, "cost_per_test": 0.165802, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "mistralai/magistral-medium-2509": { "accuracy": 4.7619, "latency": 276.9183, "stderr": 4.6471, "cost_per_test": 0.635528, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-large-2512": { "accuracy": 4.7619, "latency": 559.2309, "stderr": 4.6471, "cost_per_test": 0.222052, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.005663, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.007551, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.011326, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.014606, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.016611, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "mistralai/mistral-medium-2505": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.030698, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.061395, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/o3-mini-2025-01-31": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.083057, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2506": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.147298, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "cohere/command-a-03-2025": { "accuracy": 0.0, "latency": 47.0132, "stderr": 0.0, "cost_per_test": 0.188765, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" } }, "beat_or_miss": { "anthropic/claude-opus-4-6-thinking": { "accuracy": 51.1628, "latency": 308.0196, "stderr": 7.6229, "cost_per_test": 1.374724, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 48.8372, "latency": 110.909, "stderr": 7.6229, "cost_per_test": 2.171471, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 48.8372, "latency": 125.3426, "stderr": 7.6229, "cost_per_test": 0.766019, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 48.8372, "latency": 135.0291, "stderr": 7.6229, "cost_per_test": 2.419973, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 48.8372, "latency": 141.5414, "stderr": 7.6229, "cost_per_test": 2.470431, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 44.186, "latency": 368.3697, "stderr": 7.5732, "cost_per_test": 0.868913, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 41.8605, "latency": 46.3524, "stderr": 7.5232, "cost_per_test": 0.099316, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 41.8605, "latency": 125.0251, "stderr": 7.5232, "cost_per_test": 0.821978, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 41.8605, "latency": 178.6445, "stderr": 7.5232, "cost_per_test": 0.854557, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 41.8605, "latency": 553.3142, "stderr": 7.5232, "cost_per_test": 0.352538, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/o3-2025-04-16": { "accuracy": 39.5349, "latency": 46.3524, "stderr": 7.456, "cost_per_test": 0.064481, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 39.5349, "latency": 173.5607, "stderr": 7.456, "cost_per_test": 0.580774, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 39.5349, "latency": 616.4858, "stderr": 7.456, "cost_per_test": 0.770272, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-20250514": { "accuracy": 37.2093, "latency": 108.4739, "stderr": 7.3712, "cost_per_test": 2.326982, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "kimi/kimi-k2.5-thinking": { "accuracy": 37.2093, "latency": 297.9522, "stderr": 7.3712, "cost_per_test": 0.140227, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "google/gemini-3-flash-preview": { "accuracy": 37.2093, "latency": 1054.9358, "stderr": 7.3712, "cost_per_test": 0.350989, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "zai/glm-4.7": { "accuracy": 34.8837, "latency": 388.0001, "stderr": 7.2681, "cost_per_test": 0.156635, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 32.5581, "latency": 146.9997, "stderr": 7.146, "cost_per_test": 1.241407, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 32.5581, "latency": 152.5468, "stderr": 7.146, "cost_per_test": 0.291699, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "grok/grok-4-0709": { "accuracy": 32.5581, "latency": 227.6369, "stderr": 7.146, "cost_per_test": 0.848276, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 30.2326, "latency": 46.3524, "stderr": 7.0037, "cost_per_test": 0.035465, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 30.2326, "latency": 46.3524, "stderr": 7.0037, "cost_per_test": 0.099316, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 30.2326, "latency": 135.6804, "stderr": 7.0037, "cost_per_test": 0.232358, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 30.2326, "latency": 232.5197, "stderr": 7.0037, "cost_per_test": 0.034392, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 27.907, "latency": 46.3524, "stderr": 6.8402, "cost_per_test": 0.009067, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 27.907, "latency": 46.3524, "stderr": 6.8402, "cost_per_test": 0.099316, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-3": { "accuracy": 27.907, "latency": 46.3524, "stderr": 6.8402, "cost_per_test": 0.099316, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 27.907, "latency": 646.7312, "stderr": 6.8402, "cost_per_test": 2.688001, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 25.5814, "latency": 46.3524, "stderr": 6.6538, "cost_per_test": 0.064481, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 23.2558, "latency": 46.3524, "stderr": 6.4425, "cost_per_test": 0.044625, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 23.2558, "latency": 63.6726, "stderr": 6.4425, "cost_per_test": 0.07594, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 20.9302, "latency": 247.6514, "stderr": 6.2038, "cost_per_test": 0.062998, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-120b": { "accuracy": 20.9302, "latency": 282.075, "stderr": 6.2038, "cost_per_test": 0.108314, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 18.6047, "latency": 83.8943, "stderr": 5.9344, "cost_per_test": 0.260377, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2512": { "accuracy": 18.6047, "latency": 196.4272, "stderr": 5.9344, "cost_per_test": 0.08519, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "alibaba/qwen3-max": { "accuracy": 18.6047, "latency": 249.1839, "stderr": 5.9344, "cost_per_test": 0.397726, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "kimi/kimi-k2-thinking": { "accuracy": 13.9535, "latency": 126.1747, "stderr": 5.2841, "cost_per_test": 0.07198, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 11.6279, "latency": 84.5524, "stderr": 4.8885, "cost_per_test": 0.102092, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "zai/glm-4.6": { "accuracy": 11.6279, "latency": 163.0791, "stderr": 4.8885, "cost_per_test": 0.295891, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "google/gemini-2.0-flash-001": { "accuracy": 6.9767, "latency": 46.3524, "stderr": 3.885, "cost_per_test": 0.003224, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/o1-2024-12-17": { "accuracy": 6.9767, "latency": 46.3524, "stderr": 3.885, "cost_per_test": 0.483611, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 6.9767, "latency": 49.0246, "stderr": 3.885, "cost_per_test": 0.10368, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-preview": { "accuracy": 6.9767, "latency": 173.9413, "stderr": 3.885, "cost_per_test": 0.586278, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "minimax/MiniMax-M2.1": { "accuracy": 6.9767, "latency": 204.4162, "stderr": 3.885, "cost_per_test": 0.180572, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 4.6512, "latency": 46.3524, "stderr": 3.2115, "cost_per_test": 0.012896, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 4.6512, "latency": 46.3524, "stderr": 3.2115, "cost_per_test": 0.026484, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/o3-mini-2025-01-31": { "accuracy": 4.6512, "latency": 46.3524, "stderr": 3.2115, "cost_per_test": 0.035465, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 4.6512, "latency": 301.6969, "stderr": 3.2115, "cost_per_test": 0.566425, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 2.3256, "latency": 46.3524, "stderr": 2.2984, "cost_per_test": 0.002418, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 2.3256, "latency": 46.3524, "stderr": 2.2984, "cost_per_test": 0.004836, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-medium-2505": { "accuracy": 2.3256, "latency": 46.3524, "stderr": 2.2984, "cost_per_test": 0.013242, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2506": { "accuracy": 2.3256, "latency": 46.3524, "stderr": 2.2984, "cost_per_test": 0.061887, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 2.3256, "latency": 46.3524, "stderr": 2.2984, "cost_per_test": 0.080602, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "cohere/command-a-03-2025": { "accuracy": 2.3256, "latency": 46.3524, "stderr": 2.2984, "cost_per_test": 0.080602, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 0.0, "latency": 46.3524, "stderr": 0.0, "cost_per_test": 0.003224, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.0, "latency": 46.3524, "stderr": 0.0, "cost_per_test": 0.006102, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 0.0, "latency": 46.3524, "stderr": 0.0, "cost_per_test": 0.007093, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 0.0, "latency": 46.3524, "stderr": 0.0, "cost_per_test": 0.009067, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 0.0, "latency": 46.3524, "stderr": 0.0, "cost_per_test": 0.026089, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" } }, "trends": { "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 42.8571, "latency": 283.5648, "stderr": 10.799, "cost_per_test": 2.538427, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 42.8571, "latency": 380.5853, "stderr": 10.799, "cost_per_test": 2.938113, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 42.8571, "latency": 514.0823, "stderr": 10.799, "cost_per_test": 1.169713, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 38.0952, "latency": 150.4523, "stderr": 10.5971, "cost_per_test": 2.9455, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 38.0952, "latency": 165.4174, "stderr": 10.5971, "cost_per_test": 2.213524, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 38.0952, "latency": 168.4991, "stderr": 10.5971, "cost_per_test": 1.974223, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 38.0952, "latency": 237.238, "stderr": 10.5971, "cost_per_test": 2.57451, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "kimi/kimi-k2.5-thinking": { "accuracy": 38.0952, "latency": 538.0151, "stderr": 10.5971, "cost_per_test": 0.242737, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 33.3333, "latency": 58.4398, "stderr": 10.2869, "cost_per_test": 0.348791, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "alibaba/qwen3-max": { "accuracy": 33.3333, "latency": 596.1757, "stderr": 10.2869, "cost_per_test": 0.899115, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "google/gemini-3-flash-preview": { "accuracy": 33.3333, "latency": 690.177, "stderr": 10.2869, "cost_per_test": 0.327757, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "zai/glm-4.7": { "accuracy": 33.3333, "latency": 709.1616, "stderr": 10.2869, "cost_per_test": 0.389201, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 28.5714, "latency": 58.4398, "stderr": 9.8581, "cost_per_test": 0.348791, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 28.5714, "latency": 91.2206, "stderr": 9.8581, "cost_per_test": 0.133601, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 28.5714, "latency": 205.8974, "stderr": 9.8581, "cost_per_test": 7.484554, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 28.5714, "latency": 245.5373, "stderr": 9.8581, "cost_per_test": 8.290183, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 28.5714, "latency": 321.9269, "stderr": 9.8581, "cost_per_test": 0.131585, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 28.5714, "latency": 613.928, "stderr": 9.8581, "cost_per_test": 1.105729, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 28.5714, "latency": 925.4879, "stderr": 9.8581, "cost_per_test": 2.101006, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "openai/o3-2025-04-16": { "accuracy": 23.8095, "latency": 58.4398, "stderr": 9.2943, "cost_per_test": 0.229853, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 23.8095, "latency": 102.9234, "stderr": 9.2943, "cost_per_test": 0.497131, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-20250514": { "accuracy": 23.8095, "latency": 166.3782, "stderr": 9.2943, "cost_per_test": 8.716344, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 23.8095, "latency": 198.1294, "stderr": 9.2943, "cost_per_test": 0.723965, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 23.8095, "latency": 218.7814, "stderr": 9.2943, "cost_per_test": 9.420585, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 23.8095, "latency": 548.4526, "stderr": 9.2943, "cost_per_test": 2.137046, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 19.0476, "latency": 58.4398, "stderr": 8.5689, "cost_per_test": 0.126419, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 19.0476, "latency": 58.4398, "stderr": 8.5689, "cost_per_test": 0.348791, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 19.0476, "latency": 197.1488, "stderr": 8.5689, "cost_per_test": 0.233114, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-120b": { "accuracy": 19.0476, "latency": 329.7185, "stderr": 8.5689, "cost_per_test": 0.132486, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 14.2857, "latency": 75.9595, "stderr": 7.636, "cost_per_test": 0.291859, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-3-pro-preview": { "accuracy": 14.2857, "latency": 162.6958, "stderr": 7.636, "cost_per_test": 1.05804, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "zai/glm-4.6": { "accuracy": 14.2857, "latency": 193.6087, "stderr": 7.636, "cost_per_test": 0.406977, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "kimi/kimi-k2-thinking": { "accuracy": 14.2857, "latency": 202.2676, "stderr": 7.636, "cost_per_test": 0.194512, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "grok/grok-4-0709": { "accuracy": 14.2857, "latency": 1360.5407, "stderr": 7.636, "cost_per_test": 2.327366, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 9.5238, "latency": 58.4398, "stderr": 6.4056, "cost_per_test": 0.033542, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "openai/o3-mini-2025-01-31": { "accuracy": 9.5238, "latency": 58.4398, "stderr": 6.4056, "cost_per_test": 0.126419, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2506": { "accuracy": 9.5238, "latency": 58.4398, "stderr": 6.4056, "cost_per_test": 0.225841, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "minimax/MiniMax-M2.1": { "accuracy": 9.5238, "latency": 223.4527, "stderr": 6.4056, "cost_per_test": 0.192166, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "mistralai/mistral-large-2512": { "accuracy": 9.5238, "latency": 317.9806, "stderr": 6.4056, "cost_per_test": 0.20831, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 4.7619, "latency": 58.4398, "stderr": 4.6471, "cost_per_test": 0.008619, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "google/gemini-2.0-flash-001": { "accuracy": 4.7619, "latency": 58.4398, "stderr": 4.6471, "cost_per_test": 0.011493, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 4.7619, "latency": 58.4398, "stderr": 4.6471, "cost_per_test": 0.033542, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 4.7619, "latency": 58.4398, "stderr": 4.6471, "cost_per_test": 0.150345, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 4.7619, "latency": 58.4398, "stderr": 4.6471, "cost_per_test": 0.229853, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-3": { "accuracy": 4.7619, "latency": 58.4398, "stderr": 4.6471, "cost_per_test": 0.348791, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 4.7619, "latency": 72.2997, "stderr": 4.6471, "cost_per_test": 0.125296, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "mistralai/magistral-medium-2509": { "accuracy": 4.7619, "latency": 1018.477, "stderr": 4.6471, "cost_per_test": 0.874661, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.011493, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.017239, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.02245, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.025284, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.045971, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-medium-2505": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.046506, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.093011, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.097605, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.287316, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "cohere/command-a-03-2025": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 0.287316, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" }, "openai/o1-2024-12-17": { "accuracy": 0.0, "latency": 58.4398, "stderr": 0.0, "cost_per_test": 1.723897, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "alibaba/qwen3-max-preview": { "accuracy": 0.0, "latency": 768.4023, "stderr": 0.0, "cost_per_test": 1.149209, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" } }, "adjustments": { "anthropic/claude-opus-4-6-thinking": { "accuracy": 70.3704, "latency": 306.4913, "stderr": 8.7877, "cost_per_test": 1.480217, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 59.2593, "latency": 51.5597, "stderr": 9.4561, "cost_per_test": 0.245163, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 59.2593, "latency": 116.8285, "stderr": 9.4561, "cost_per_test": 1.240207, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 59.2593, "latency": 331.8089, "stderr": 9.4561, "cost_per_test": 0.41565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 55.5556, "latency": 51.5597, "stderr": 9.5629, "cost_per_test": 0.245163, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 55.5556, "latency": 118.9053, "stderr": 9.5629, "cost_per_test": 0.923611, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 51.8519, "latency": 341.1798, "stderr": 9.6159, "cost_per_test": 0.903068, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "xhigh", "verbosity": null, "provider": "OpenAI" }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 51.8519, "latency": 488.8851, "stderr": 9.6159, "cost_per_test": 1.042324, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 48.1481, "latency": 72.0303, "stderr": 9.6159, "cost_per_test": 1.323016, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 48.1481, "latency": 92.0413, "stderr": 9.6159, "cost_per_test": 0.920748, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 48.1481, "latency": 162.0534, "stderr": 9.6159, "cost_per_test": 4.318502, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 48.1481, "latency": 207.4103, "stderr": 9.6159, "cost_per_test": 0.085456, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "openai/o3-2025-04-16": { "accuracy": 44.4444, "latency": 51.5597, "stderr": 9.5629, "cost_per_test": 0.161384, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 44.4444, "latency": 51.5597, "stderr": 9.5629, "cost_per_test": 0.245163, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 44.4444, "latency": 108.4174, "stderr": 9.5629, "cost_per_test": 4.408361, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 44.4444, "latency": 110.2348, "stderr": 9.5629, "cost_per_test": 4.188658, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "google/gemini-3-pro-preview": { "accuracy": 44.4444, "latency": 148.3721, "stderr": 9.5629, "cost_per_test": 0.665702, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "kimi/kimi-k2.5-thinking": { "accuracy": 44.4444, "latency": 243.7519, "stderr": 9.5629, "cost_per_test": 0.158073, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "grok/grok-4-0709": { "accuracy": 44.4444, "latency": 339.5354, "stderr": 9.5629, "cost_per_test": 1.195896, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "zai/glm-4.7": { "accuracy": 44.4444, "latency": 463.7706, "stderr": 9.5629, "cost_per_test": 0.251572, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "openai/gpt-5-2025-08-07": { "accuracy": 44.4444, "latency": 477.7657, "stderr": 9.5629, "cost_per_test": 0.70262, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 40.7407, "latency": 51.5597, "stderr": 9.4561, "cost_per_test": 0.088761, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 39.2857, "latency": 198.9473, "stderr": 9.2296, "cost_per_test": 0.07035, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 37.037, "latency": 41.8529, "stderr": 9.2935, "cost_per_test": 0.055462, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 37.037, "latency": 58.1554, "stderr": 9.2935, "cost_per_test": 0.066848, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "grok/grok-4-fast-reasoning": { "accuracy": 37.037, "latency": 73.0413, "stderr": 9.2935, "cost_per_test": 0.200624, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 37.037, "latency": 85.5556, "stderr": 9.2935, "cost_per_test": 0.846388, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 37.037, "latency": 97.9053, "stderr": 9.2935, "cost_per_test": 0.261251, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 37.037, "latency": 134.2463, "stderr": 9.2935, "cost_per_test": 3.796587, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "minimax/MiniMax-M2.1": { "accuracy": 37.037, "latency": 168.5099, "stderr": 9.2935, "cost_per_test": 0.107269, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning_effort": null, "verbosity": null, "provider": "MiniMax" }, "alibaba/qwen3-max": { "accuracy": 37.037, "latency": 250.7993, "stderr": 9.2935, "cost_per_test": 0.33783, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 33.3333, "latency": 51.5597, "stderr": 9.0722, "cost_per_test": 0.023487, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "high", "verbosity": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 29.6296, "latency": 51.5597, "stderr": 8.7877, "cost_per_test": 0.032277, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 29.6296, "latency": 51.5597, "stderr": 8.7877, "cost_per_test": 0.10601, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "google/gemini-3-flash-preview": { "accuracy": 29.6296, "latency": 1102.6367, "stderr": 8.7877, "cost_per_test": 0.612324, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning_effort": "high", "verbosity": null, "provider": "Google" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 25.9259, "latency": 41.4048, "stderr": 8.4337, "cost_per_test": 0.134509, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 22.2222, "latency": 51.5597, "stderr": 8.0009, "cost_per_test": 0.023487, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": "low", "verbosity": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-120b": { "accuracy": 22.2222, "latency": 108.6108, "stderr": 8.0009, "cost_per_test": 0.054847, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "kimi/kimi-k2-thinking": { "accuracy": 22.2222, "latency": 167.3631, "stderr": 8.0009, "cost_per_test": 0.11867, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Moonshot AI" }, "grok/grok-3": { "accuracy": 18.5185, "latency": 51.5597, "stderr": 7.4757, "cost_per_test": 0.245163, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning_effort": null, "verbosity": null, "provider": "SpaceXAI" }, "zai/glm-4.6": { "accuracy": 18.5185, "latency": 163.2254, "stderr": 7.4757, "cost_per_test": 0.270515, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning_effort": null, "verbosity": null, "provider": "Zhipu AI" }, "mistralai/mistral-large-2512": { "accuracy": 18.5185, "latency": 521.5073, "stderr": 7.4757, "cost_per_test": 0.195905, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "google/gemini-2.0-flash-001": { "accuracy": 14.8148, "latency": 51.5597, "stderr": 6.8367, "cost_per_test": 0.008069, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Google" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 14.8148, "latency": 51.5597, "stderr": 6.8367, "cost_per_test": 0.065377, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Anthropic" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 14.8148, "latency": 51.5597, "stderr": 6.8367, "cost_per_test": 0.161384, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "openai/o1-2024-12-17": { "accuracy": 14.8148, "latency": 51.5597, "stderr": 6.8367, "cost_per_test": 1.210382, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "alibaba/qwen3-max-preview": { "accuracy": 14.8148, "latency": 204.2648, "stderr": 6.8367, "cost_per_test": 0.229874, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning_effort": null, "verbosity": null, "provider": "Alibaba" }, "openai/gpt-4o-2024-08-06": { "accuracy": 11.1111, "latency": 51.5597, "stderr": 6.0481, "cost_per_test": 0.20173, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-small-2503": { "accuracy": 7.4074, "latency": 51.5597, "stderr": 5.0401, "cost_per_test": 0.006052, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 3.7037, "latency": 51.5597, "stderr": 3.6345, "cost_per_test": 0.012104, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "mistralai/mistral-medium-2505": { "accuracy": 3.7037, "latency": 51.5597, "stderr": 3.6345, "cost_per_test": 0.032688, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2506": { "accuracy": 3.7037, "latency": 51.5597, "stderr": 3.6345, "cost_per_test": 0.158298, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "mistralai/magistral-medium-2509": { "accuracy": 3.7037, "latency": 654.0939, "stderr": 3.6345, "cost_per_test": 0.734099, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning_effort": null, "verbosity": null, "provider": "Mistral AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 0.0, "latency": 51.5597, "stderr": 0.0, "cost_per_test": 0.008069, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.0, "latency": 51.5597, "stderr": 0.0, "cost_per_test": 0.015727, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "AI21 Labs" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 0.0, "latency": 51.5597, "stderr": 0.0, "cost_per_test": 0.017752, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning_effort": null, "verbosity": null, "provider": "Fireworks AI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 0.0, "latency": 51.5597, "stderr": 0.0, "cost_per_test": 0.068293, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning_effort": null, "verbosity": null, "provider": "Together AI" }, "openai/o3-mini-2025-01-31": { "accuracy": 0.0, "latency": 51.5597, "stderr": 0.0, "cost_per_test": 0.088761, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning_effort": "high", "verbosity": null, "provider": "OpenAI" }, "cohere/command-a-03-2025": { "accuracy": 0.0, "latency": 51.5597, "stderr": 0.0, "cost_per_test": 0.20173, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning_effort": null, "verbosity": null, "provider": "Cohere" } } } }