{ "metadata": { "benchmark": "CaseLaw v2", "slug": "case_law_v2", "description": "Private question-answer benchmark over Canadian court-cases.", "benchmark_id": "case_law_v2", "family": "case_law", "version": "2", "updated": "2026-05-04", "dataset_type": "private", "industry": "legal", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-max-preview", "alibaba/qwen3.6-plus", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "arcee-ai/trinity-large-thinking", "cohere/command-a-03-2025", "deepseek/deepseek-v4-pro", "fireworks/deepseek-v3p1", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemma-4-31b-it", "grok/grok-4-0709", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "meta/muse_spark", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "mistralai/mistral-large-2512", "mistralai/mistral-medium-3.5", "openai/gpt-4.1-2025-04-14", "openai/gpt-4o-2024-11-20", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1" ], "partners": [ { "name": "jurisage", "url": "https://jurisage.com" }, { "name": "tai", "url": "https://www.together.ai/" }, { "name": "codex", "url": "https://law.stanford.edu/codex-the-stanford-center-for-legal-informatics/" } ], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": true, "partner": false, "total_models": 54 }, "tasks": { "overall": { "grok/grok-4.3": { "accuracy": 79.314, "latency": 42.813, "stderr": 1.368, "cost_per_test": 0.040307, "avg_input_tokens": 34957.3, "avg_output_tokens": 2134.3, "avg_reasoning_tokens": 1541.6, "avg_cache_read_tokens": 19791.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 42.8, "median_query_seconds": 53.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 73.419, "latency": 23.591, "stderr": 0.75, "cost_per_test": 0.042966, "avg_input_tokens": 35481.8, "avg_output_tokens": 1264.4, "avg_reasoning_tokens": 919.7, "avg_cache_read_tokens": 12471.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 23.6, "median_query_seconds": 12.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 69.882, "latency": 6.656, "stderr": 0.284, "cost_per_test": 0.046588, "avg_input_tokens": 35765.2, "avg_output_tokens": 300.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 18233.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 68.489, "latency": 42.506, "stderr": 1.129, "cost_per_test": 0.011305, "avg_input_tokens": 35481.8, "avg_output_tokens": 2641.3, "avg_reasoning_tokens": 2328.8, "avg_cache_read_tokens": 12657.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 42.5, "median_query_seconds": 29.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 68.381, "latency": 17.677, "stderr": 0.429, "cost_per_test": 0.161535, "avg_input_tokens": 53236.3, "avg_output_tokens": 1314.7, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 17.7, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 66.452, "latency": 123.423, "stderr": 0.652, "cost_per_test": 0.066591, "avg_input_tokens": 35481.8, "avg_output_tokens": 2331.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 123.4, "median_query_seconds": 101.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 66.238, "latency": 25.2, "stderr": 0.322, "cost_per_test": 0.155011, "avg_input_tokens": 35272.7, "avg_output_tokens": 1183.3, "avg_reasoning_tokens": 980.3, "avg_cache_read_tokens": 12633.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 25.2, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 66.024, "latency": 86.227, "stderr": 0.567, "cost_per_test": 0.103778, "avg_input_tokens": 35481.9, "avg_output_tokens": 3268.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 86.2, "median_query_seconds": 51.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 65.809, "latency": 32.335, "stderr": 0.214, "cost_per_test": 0.158935, "avg_input_tokens": 34163.6, "avg_output_tokens": 1071.1, "avg_reasoning_tokens": 814.8, "avg_cache_read_tokens": 7530.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 32.3, "median_query_seconds": 20.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 65.702, "latency": 23.709, "stderr": 0.429, "cost_per_test": 0.007292, "avg_input_tokens": 35100.3, "avg_output_tokens": 950.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 23.7, "median_query_seconds": 14.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 65.702, "latency": 91.045, "stderr": 1.822, "cost_per_test": 0.015284, "avg_input_tokens": 29706.5, "avg_output_tokens": 2138.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 17526.8, "turn_metrics": { "tasks_measured": 100, "avg_turns": 1.0, "avg_query_seconds": 91.1, "median_query_seconds": 74.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 64.845, "latency": 56.785, "stderr": 1.236, "cost_per_test": 0.136185, "avg_input_tokens": 38365.0, "avg_output_tokens": 1789.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 56.8, "median_query_seconds": 38.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 64.523, "latency": 41.555, "stderr": 1.088, "cost_per_test": 0.098244, "avg_input_tokens": 37853.9, "avg_output_tokens": 360.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 41.6, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 63.987, "latency": 36.196, "stderr": 0.557, "cost_per_test": 0.144942, "avg_input_tokens": 33221.6, "avg_output_tokens": 1357.5, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 36.2, "median_query_seconds": 20.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 63.88, "latency": 44.02, "stderr": 0.597, "cost_per_test": 0.257186, "avg_input_tokens": 40231.4, "avg_output_tokens": 1446.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 44.0, "median_query_seconds": 25.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 63.773, "latency": 56.797, "stderr": 2.238, "cost_per_test": 0.100765, "avg_input_tokens": 35764.2, "avg_output_tokens": 2141.5, "avg_reasoning_tokens": 1939.6, "avg_cache_read_tokens": 9229.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 56.8, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 63.13, "latency": 61.867, "stderr": 0.75, "cost_per_test": 0.006504, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 62.594, "latency": 16.819, "stderr": 0.107, "cost_per_test": 0.229072, "avg_input_tokens": 32813.0, "avg_output_tokens": 963.1, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 16.8, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 62.165, "latency": 25.416, "stderr": 1.546, "cost_per_test": 0.255566, "avg_input_tokens": 40231.4, "avg_output_tokens": 1372.8, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 25.4, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 62.058, "latency": 32.273, "stderr": 0.371, "cost_per_test": 0.231218, "avg_input_tokens": 33221.6, "avg_output_tokens": 946.4, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 32.3, "median_query_seconds": 24.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 61.415, "latency": 40.839, "stderr": 0.965, "cost_per_test": 0.018968, "avg_input_tokens": 36744.6, "avg_output_tokens": 397.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 40.8, "median_query_seconds": 27.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 61.201, "latency": 46.087, "stderr": 2.087, "cost_per_test": 0.025569, "avg_input_tokens": 32697.9, "avg_output_tokens": 2324.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 18724.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 101, "avg_turns": 1.0, "avg_query_seconds": 46.1, "median_query_seconds": 29.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 60.879, "latency": 21.639, "stderr": 0.773, "cost_per_test": 0.003091, "avg_input_tokens": 26190.9, "avg_output_tokens": 690.6, "avg_cache_read_tokens": 21911.5, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 21.6, "median_query_seconds": 15.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 60.45, "latency": 19.094, "stderr": 0.809, "cost_per_test": 0.011453, "avg_input_tokens": 35100.3, "avg_output_tokens": 962.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 19.1, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 59.7, "latency": 40.867, "stderr": 0.429, "cost_per_test": 0.066774, "avg_input_tokens": 26096.7, "avg_output_tokens": 247.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 40.9, "median_query_seconds": 30.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 59.7, "latency": 94.691, "stderr": 1.134, "cost_per_test": 0.029604, "avg_input_tokens": 30987.4, "avg_output_tokens": 4187.1, "turn_metrics": { "tasks_measured": 101, "avg_turns": 1.0, "avg_query_seconds": 94.7, "median_query_seconds": 79.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 59.378, "latency": 48.444, "stderr": 1.5, "cost_per_test": 0.013152, "avg_input_tokens": 36455.9, "avg_output_tokens": 2229.4, "avg_reasoning_tokens": 2062.1, "avg_cache_read_tokens": 36388.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 48.4, "median_query_seconds": 24.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 58.735, "latency": 241.278, "stderr": 0.703, "cost_per_test": 0.024886, "avg_input_tokens": 33943.8, "avg_output_tokens": 1555.0, "turn_metrics": { "tasks_measured": 103, "avg_turns": 1.0, "avg_query_seconds": 241.3, "median_query_seconds": 221.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 57.878, "latency": 14.554, "stderr": 1.856, "cost_per_test": 0.008867, "avg_input_tokens": 33836.4, "avg_output_tokens": 1060.6, "avg_reasoning_tokens": 587.4, "avg_cache_read_tokens": 1897.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 14.6, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 56.484, "latency": 32.919, "stderr": 0.916, "cost_per_test": 0.049552, "avg_input_tokens": 33113.8, "avg_output_tokens": 1635.4, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 32.9, "median_query_seconds": 19.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 55.948, "latency": 43.543, "stderr": 1.159, "cost_per_test": 0.004909, "avg_input_tokens": 32263.1, "avg_output_tokens": 4206.0, "avg_reasoning_tokens": 3960.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 101, "avg_turns": 1.0, "avg_query_seconds": 43.5, "median_query_seconds": 36.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 55.842, "latency": 37.357, "stderr": 0.467, "cost_per_test": 0.008008, "avg_input_tokens": 26302.0, "avg_output_tokens": 980.6, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 37.3, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 55.842, "latency": 260.502, "stderr": 0.107, "cost_per_test": 0.025399, "avg_input_tokens": 38365.0, "avg_output_tokens": 2090.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 260.5, "median_query_seconds": 276.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 55.412, "latency": 66.077, "stderr": 0.467, "cost_per_test": 0.017751, "avg_input_tokens": 28985.4, "avg_output_tokens": 1003.9, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 66.1, "median_query_seconds": 41.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 54.984, "latency": 4.641, "stderr": 0.742, "cost_per_test": 0.009312, "avg_input_tokens": 38369.4, "avg_output_tokens": 1498.0, "avg_reasoning_tokens": 1292.8, "avg_cache_read_tokens": 11232.5, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 54.984, "latency": 133.865, "stderr": 1.485, "cost_per_test": 0.085444, "avg_input_tokens": 25564.3, "avg_output_tokens": 3966.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 99, "avg_turns": 1.0, "avg_query_seconds": 133.9, "median_query_seconds": 112.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.7": { "accuracy": 54.877, "latency": 59.643, "stderr": 0.916, "cost_per_test": 0.021159, "avg_input_tokens": 27668.0, "avg_output_tokens": 2201.3, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 59.7, "median_query_seconds": 39.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 54.448, "latency": 10.663, "stderr": 0.284, "cost_per_test": 0.068241, "avg_input_tokens": 37499.5, "avg_output_tokens": 1419.8, "avg_reasoning_tokens": 1244.6, "avg_cache_read_tokens": 20767.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 10.7, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 53.912, "latency": 40.682, "stderr": 0.567, "cost_per_test": 0.016723, "avg_input_tokens": 29259.6, "avg_output_tokens": 201.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 40.7, "median_query_seconds": 28.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 53.483, "latency": 30.875, "stderr": 0.467, "cost_per_test": 0.012109, "avg_input_tokens": 26194.5, "avg_output_tokens": 957.6, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 30.9, "median_query_seconds": 21.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 53.162, "latency": 42.493, "stderr": 2.168, "cost_per_test": 0.023877, "avg_input_tokens": 25587.0, "avg_output_tokens": 2368.1, "avg_reasoning_tokens": 2161.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 99, "avg_turns": 1.0, "avg_query_seconds": 42.5, "median_query_seconds": 33.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 53.055, "latency": 32.345, "stderr": 0.186, "cost_per_test": 0.137729, "avg_input_tokens": 38365.0, "avg_output_tokens": 1918.7, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 32.3, "median_query_seconds": 22.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 52.626, "latency": 28.716, "stderr": 1.343, "cost_per_test": 0.0, "avg_input_tokens": 36026.3, "avg_output_tokens": 1051.7, "avg_reasoning_tokens": 890.6, "avg_cache_read_tokens": 2559.3, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 28.7, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 52.626, "latency": 56.48, "stderr": 1.022, "cost_per_test": 0.004165, "avg_input_tokens": 35481.8, "avg_output_tokens": 6002.5, "avg_reasoning_tokens": 5755.7, "avg_cache_read_tokens": 217.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 56.5, "median_query_seconds": 47.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 52.519, "latency": 117.983, "stderr": 1.725, "cost_per_test": 0.0344, "avg_input_tokens": 27080.6, "avg_output_tokens": 2360.1, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 118.0, "median_query_seconds": 84.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 51.875, "latency": 7.62, "stderr": 1.088, "cost_per_test": 0.001108, "avg_input_tokens": 35481.8, "avg_output_tokens": 641.2, "avg_reasoning_tokens": 414.7, "avg_cache_read_tokens": 20512.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 51.661, "latency": 57.683, "stderr": 0.934, "cost_per_test": 0.040877, "avg_input_tokens": 34838.0, "avg_output_tokens": 6318.5, "avg_reasoning_tokens": 6173.2, "avg_cache_read_tokens": 20273.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 57.7, "median_query_seconds": 29.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 51.554, "latency": 48.961, "stderr": 1.134, "cost_per_test": 0.034033, "avg_input_tokens": 31631.6, "avg_output_tokens": 2114.1, "avg_reasoning_tokens": 1954.7, "avg_cache_read_tokens": 5455.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 49.0, "median_query_seconds": 29.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 51.447, "latency": 43.909, "stderr": 1.159, "cost_per_test": 0.047281, "avg_input_tokens": 38313.7, "avg_output_tokens": 2342.7, "avg_reasoning_tokens": 2167.8, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 43.9, "median_query_seconds": 39.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 48.767, "latency": 68.348, "stderr": 0.877, "cost_per_test": 0.004339, "avg_input_tokens": 26193.5, "avg_output_tokens": 683.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 68.4, "median_query_seconds": 48.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 47.91, "latency": 42.93, "stderr": 1.114, "cost_per_test": 0.05235, "avg_input_tokens": 24287.5, "avg_output_tokens": 1997.0, "avg_reasoning_tokens": 1809.1, "turn_metrics": { "tasks_measured": 98, "avg_turns": 1.0, "avg_query_seconds": 42.9, "median_query_seconds": 39.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 47.481, "latency": 84.725, "stderr": 0.467, "cost_per_test": 0.050593, "avg_input_tokens": 23878.0, "avg_output_tokens": 202.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 98, "avg_turns": 1.0, "avg_query_seconds": 84.7, "median_query_seconds": 63.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 44.159, "latency": 64.635, "stderr": 1.056, "cost_per_test": 0.084987, "avg_input_tokens": 36178.7, "avg_output_tokens": 4095.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 64.6, "median_query_seconds": 57.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 43.837, "latency": 85.696, "stderr": 0.935, "cost_per_test": 0.002158, "avg_input_tokens": 26098.4, "avg_output_tokens": 1105.7, "turn_metrics": { "tasks_measured": 104, "avg_turns": 1.0, "avg_query_seconds": 85.7, "median_query_seconds": 59.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }