{ "metadata": { "benchmark": "Terminal-Bench-Science", "slug": "terminal-bench-science", "description": "Research workflow tasks contributed by practicing scientists across five domains", "benchmark_id": "terminal_bench_science", "family": "terminal_bench_science", "version": "0.1", "updated": "2026-10-25", "dataset_type": "public", "industry": "science", "tasks": { "overall": "Overall", "life_sciences": "Life Sciences", "physical_sciences": "Physical Sciences", "earth_sciences": "Earth Sciences", "mathematical_sciences": "Mathematical Sciences", "engineering_sciences": "Engineering Sciences" }, "models": [ "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "fireworks/ember-1", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k3", "kimi/kimi-k3.1-pro", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "reflection/Beam-1.0", "stepfun/step-5-preview", "tencent/hy4-preview", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 70, "harness": "Mini-SWE-agent", "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 43 }, "tasks": { "overall": { "openai/gpt-6-astra": { "accuracy": 62.857, "latency": 6534.868, "stderr": 5.817, "cost_per_test": 20.802802, "token_totals": { "input_tokens": 405026379, "output_tokens": 11095298, "reasoning_tokens": 8544230, "cache_read_tokens": 386366671, "cache_write_tokens": 18635815 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 47.1, "avg_query_seconds": 120.1, "median_query_seconds": 65.1, "avg_ttft_seconds": 98.6, "output_time_split": { "reasoning_pct": 2.9, "content_pct": 18.6, "tool_call_pct": 78.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6.1-sol": { "accuracy": 52.857, "latency": 5003.987, "stderr": 6.009, "cost_per_test": 3.436948, "token_totals": { "input_tokens": 337815176, "output_tokens": 11533409, "reasoning_tokens": 9185469, "cache_read_tokens": 317832491, "cache_write_tokens": 19972804 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 42.1, "avg_query_seconds": 130.2, "median_query_seconds": 61.4, "avg_ttft_seconds": 106.4, "output_time_split": { "reasoning_pct": 4.0, "content_pct": 11.1, "tool_call_pct": 84.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5-5": { "accuracy": 47.143, "latency": 8856.116, "stderr": 6.009, "cost_per_test": 19.121966, "token_totals": { "input_tokens": 1829311051, "output_tokens": 17393519, "reasoning_tokens": 11843007, "cache_read_tokens": 1700954981, "cache_write_tokens": 128339444 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 117.3, "avg_query_seconds": 22.9, "median_query_seconds": 14.2, "avg_ttft_seconds": 21.8, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5-5": { "accuracy": 47.143, "latency": 9980.148, "stderr": 6.009, "cost_per_test": 19.568918, "token_totals": { "input_tokens": 9111131259, "output_tokens": 34342805, "reasoning_tokens": 24263221, "cache_read_tokens": 9063088192, "cache_write_tokens": 48002507 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 285.1, "avg_query_seconds": 16.6, "median_query_seconds": 10.6, "avg_ttft_seconds": 16.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "google/gemini-4-argon": { "accuracy": 44.286, "latency": 6369.095, "stderr": 5.98, "cost_per_test": 44.904641, "token_totals": { "input_tokens": 2230737485, "output_tokens": 21755185, "reasoning_tokens": 10397231, "cache_read_tokens": 1635454933, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 113.1, "avg_query_seconds": 47.3, "median_query_seconds": 35.2, "avg_ttft_seconds": 14.4, "output_time_split": { "reasoning_pct": 52.8, "content_pct": 46.9, "tool_call_pct": 0.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5-1": { "accuracy": 40.0, "latency": 9376.226, "stderr": 5.898, "cost_per_test": 38.010073, "token_totals": { "input_tokens": 1408818997, "output_tokens": 15104550, "reasoning_tokens": 9261638, "cache_read_tokens": 1275387088, "cache_write_tokens": 133417273 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 105.8, "avg_query_seconds": 28.9, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "meta/muse-wonder-full": { "accuracy": 37.143, "latency": 12919.209, "stderr": 5.817, "cost_per_test": 22.814086, "token_totals": { "input_tokens": 3713083934, "output_tokens": 17062678, "reasoning_tokens": 11556867, "cache_read_tokens": 3578053496, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 191.6, "avg_query_seconds": 54.4, "median_query_seconds": 20.5, "avg_ttft_seconds": 52.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3.1-pro": { "accuracy": 31.429, "latency": 3474.879, "stderr": 5.589, "cost_per_test": 8.577155, "token_totals": { "input_tokens": 2310199935, "output_tokens": 10019167, "reasoning_tokens": 5787120, "cache_read_tokens": 2288994816, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "openai/gpt-6-sol": { "accuracy": 30.0, "latency": 4930.428, "stderr": 5.517, "cost_per_test": 5.821477, "token_totals": { "input_tokens": 1106517592, "output_tokens": 9142533, "reasoning_tokens": 5482665, "cache_read_tokens": 1090637986, "cache_write_tokens": 15843344 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 105.6, "avg_query_seconds": 26.1, "median_query_seconds": 15.6, "avg_ttft_seconds": 13.7, "output_time_split": { "reasoning_pct": 12.8, "content_pct": 5.8, "tool_call_pct": 81.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5": { "accuracy": 27.143, "latency": 7908.453, "stderr": 5.354, "cost_per_test": 32.543386, "token_totals": { "input_tokens": 3251326101, "output_tokens": 18648623, "reasoning_tokens": 11809096, "cache_read_tokens": 3218944868, "cache_write_tokens": 32354267 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 192.6, "avg_query_seconds": 20.2, "median_query_seconds": 9.2, "avg_ttft_seconds": 19.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "meta/muse-wonder": { "accuracy": 22.857, "latency": 5584.198, "stderr": 5.055, "cost_per_test": 19.609644, "token_totals": { "input_tokens": 4434041175, "output_tokens": 17847376, "reasoning_tokens": 12044271, "cache_read_tokens": 4400115036, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 238.7, "avg_query_seconds": 13.5, "median_query_seconds": 7.1, "avg_ttft_seconds": 12.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-sol": { "accuracy": 20.0, "latency": 6216.027, "stderr": 4.815, "cost_per_test": 7.322071, "token_totals": { "input_tokens": 662422169, "output_tokens": 6024299, "reasoning_tokens": 2789570, "cache_read_tokens": 648354535, "cache_write_tokens": 14038242 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 72.8, "avg_query_seconds": 19.7, "median_query_seconds": 13.1, "avg_ttft_seconds": 10.0, "output_time_split": { "reasoning_pct": 14.9, "content_pct": 26.8, "tool_call_pct": 58.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5": { "accuracy": 15.714, "latency": 7157.669, "stderr": 4.381, "cost_per_test": 51.990885, "token_totals": { "input_tokens": 2599459585, "output_tokens": 16914583, "reasoning_tokens": 10535367, "cache_read_tokens": 2569404286, "cache_write_tokens": 30031671 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 166.5, "avg_query_seconds": 24.6, "median_query_seconds": 9.6, "avg_ttft_seconds": 21.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-max": { "accuracy": 12.857, "latency": 12243.338, "stderr": 4.03, "cost_per_test": 16.682189, "token_totals": { "input_tokens": 3844140557, "output_tokens": 22910070, "reasoning_tokens": 16758796, "cache_read_tokens": 3804564736, "cache_write_tokens": null }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 185.4, "avg_query_seconds": 49.2, "median_query_seconds": 31.6, "avg_ttft_seconds": 10.9, "output_time_split": { "reasoning_pct": 77.2, "content_pct": 0.3, "tool_call_pct": 22.5 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "grok/grok-4.6": { "accuracy": 11.429, "latency": 4696.051, "stderr": 3.83, "cost_per_test": 4.815933, "token_totals": { "input_tokens": 519819247, "output_tokens": 7158015, "reasoning_tokens": 3485074, "cache_read_tokens": 496980864, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 65.2, "avg_query_seconds": 24.2, "median_query_seconds": 13.9, "avg_ttft_seconds": 4.8, "output_time_split": { "reasoning_pct": 79.3, "content_pct": 20.7, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3_max": { "accuracy": 10.0, "latency": 5045.791, "stderr": 3.612, "cost_per_test": 6.094983, "token_totals": { "input_tokens": 1998607677, "output_tokens": 19369661, "reasoning_tokens": 12665202, "cache_read_tokens": 1958119889, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 132.3, "avg_query_seconds": 13.5, "median_query_seconds": 8.9, "avg_ttft_seconds": 6.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "grok/grok-4.7": { "accuracy": 10.0, "latency": 5553.191, "stderr": 3.612, "cost_per_test": 17.040638, "token_totals": { "input_tokens": 1158107075, "output_tokens": 11957556, "reasoning_tokens": 6886200, "cache_read_tokens": 1084056704, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 93.7, "avg_query_seconds": 26.6, "median_query_seconds": 18.4, "avg_ttft_seconds": 3.7, "output_time_split": { "reasoning_pct": 94.4, "content_pct": 5.6, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-haiku-5-5": { "accuracy": 10.0, "latency": 5637.472, "stderr": 3.612, "cost_per_test": 4.523698, "token_totals": { "input_tokens": 4964740201, "output_tokens": 26152873, "reasoning_tokens": 19018280, "cache_read_tokens": 4931170604, "cache_write_tokens": 33540337 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 208.4, "avg_query_seconds": 9.0, "median_query_seconds": 4.1, "avg_ttft_seconds": 7.6, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-terra": { "accuracy": 10.0, "latency": 7443.23, "stderr": 3.612, "cost_per_test": 5.19351, "token_totals": { "input_tokens": 935527799, "output_tokens": 7338523, "reasoning_tokens": 4151458, "cache_read_tokens": 919964318, "cache_write_tokens": 15421007 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 96.4, "avg_query_seconds": 19.3, "median_query_seconds": 12.4, "avg_ttft_seconds": 9.1, "output_time_split": { "reasoning_pct": 25.6, "content_pct": 8.9, "tool_call_pct": 65.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "google/gemini-3.8-flash": { "accuracy": 8.571, "latency": 3311.435, "stderr": 3.37, "cost_per_test": 5.643008, "token_totals": { "input_tokens": 1352310020, "output_tokens": 11150231, "reasoning_tokens": 5862209, "cache_read_tokens": 1271911976, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 69, "avg_turns": 146.3, "avg_query_seconds": 5.8, "median_query_seconds": 4.2, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 97.9, "content_pct": 0.9, "tool_call_pct": 1.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.7-flash": { "accuracy": 7.143, "latency": 4825.144, "stderr": 3.1, "cost_per_test": 7.190413, "token_totals": { "input_tokens": 1286456118, "output_tokens": 10695665, "reasoning_tokens": 3697777, "cache_read_tokens": 1115979803, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 109.8, "avg_query_seconds": 6.7, "median_query_seconds": 5.1, "avg_ttft_seconds": 4.3, "output_time_split": { "reasoning_pct": 96.5, "content_pct": 2.0, "tool_call_pct": 1.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.5-flash": { "accuracy": 5.714, "latency": 3654.61, "stderr": 2.794, "cost_per_test": 3.802646, "token_totals": { "input_tokens": 634004951, "output_tokens": 8326111, "reasoning_tokens": 4021440, "cache_read_tokens": 562783101, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 83.1, "avg_query_seconds": 8.5, "median_query_seconds": 4.8, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 57.6, "content_pct": 41.8, "tool_call_pct": 0.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "zai/glm-5.3": { "accuracy": 5.714, "latency": 9898.858, "stderr": 2.794, "cost_per_test": 15.627955, "token_totals": { "input_tokens": 3692684680, "output_tokens": 18182694, "reasoning_tokens": 13144712, "cache_read_tokens": 3645443456, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 220.6, "avg_query_seconds": 25.9, "median_query_seconds": 12.6, "avg_ttft_seconds": 6.9, "output_time_split": { "reasoning_pct": 99.1, "content_pct": 0.9, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5": { "accuracy": 5.714, "latency": 11157.924, "stderr": 2.794, "cost_per_test": 30.299862, "token_totals": { "input_tokens": 8555375360, "output_tokens": 26718888, "reasoning_tokens": 18294700, "cache_read_tokens": 8493310382, "cache_write_tokens": 62018810 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 333.3, "avg_query_seconds": 16.1, "median_query_seconds": 7.7, "avg_ttft_seconds": 15.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "fireworks/ember-1": { "accuracy": 5.714, "latency": 12196.691, "stderr": 2.794, "cost_per_test": 30.760832, "token_totals": { "input_tokens": 5574642539, "output_tokens": 19999519, "reasoning_tokens": 13579586, "cache_read_tokens": 5507652660, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 276.7, "avg_query_seconds": 23.4, "median_query_seconds": 13.2, "avg_ttft_seconds": 6.4, "output_time_split": { "reasoning_pct": 71.1, "content_pct": 1.7, "tool_call_pct": 27.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3": { "accuracy": 4.286, "latency": 4050.414, "stderr": 2.438, "cost_per_test": 4.407744, "token_totals": { "input_tokens": 1454427801, "output_tokens": 14032439, "reasoning_tokens": 8560708, "cache_read_tokens": 1426482303, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 124.8, "avg_query_seconds": 9.5, "median_query_seconds": 6.3, "avg_ttft_seconds": 5.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-6-luna": { "accuracy": 4.286, "latency": 4559.47, "stderr": 2.438, "cost_per_test": 0.240966, "token_totals": { "input_tokens": 771110577, "output_tokens": 9956631, "reasoning_tokens": 7497116, "cache_read_tokens": 755113085, "cache_write_tokens": 15962745 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 76.0, "avg_query_seconds": 20.7, "median_query_seconds": 10.9, "avg_ttft_seconds": 7.2, "output_time_split": { "reasoning_pct": 47.6, "content_pct": 5.1, "tool_call_pct": 47.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 4.286, "latency": 5653.642, "stderr": 2.438, "cost_per_test": 1.095543, "token_totals": { "input_tokens": 3628605043, "output_tokens": 17032330, "reasoning_tokens": 12139861, "cache_read_tokens": 3620612352, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 228.5, "avg_query_seconds": 7.4, "median_query_seconds": 4.9, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 81.3, "content_pct": 0.5, "tool_call_pct": 18.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "google/gemini-3.6-flash": { "accuracy": 4.286, "latency": 6744.655, "stderr": 2.438, "cost_per_test": 4.390771, "token_totals": { "input_tokens": 906982870, "output_tokens": 10616369, "reasoning_tokens": 3143772, "cache_read_tokens": 839068957, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 69, "avg_turns": 98.1, "avg_query_seconds": 9.2, "median_query_seconds": 6.2, "avg_ttft_seconds": 3.2, "output_time_split": { "reasoning_pct": 90.0, "content_pct": 9.4, "tool_call_pct": 0.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-4-8": { "accuracy": 4.286, "latency": 6906.538, "stderr": 2.438, "cost_per_test": 23.141733, "token_totals": { "input_tokens": 1809454503, "output_tokens": 20000133, "reasoning_tokens": 14489581, "cache_read_tokens": 1772026517, "cache_write_tokens": 37411842 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 115.3, "avg_query_seconds": 35.7, "median_query_seconds": 20.0, "avg_ttft_seconds": 29.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 4.286, "latency": 8181.756, "stderr": 2.438, "cost_per_test": 4.630215, "token_totals": { "input_tokens": 5726611218, "output_tokens": 14168265, "reasoning_tokens": 6714937, "cache_read_tokens": 5714042368, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 353.2, "avg_query_seconds": 9.0, "median_query_seconds": 5.9, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 67.1, "content_pct": 0.5, "tool_call_pct": 32.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 4.286, "latency": 14326.966, "stderr": 2.438, "cost_per_test": 0.244346, "token_totals": { "input_tokens": 2120700869, "output_tokens": 19847075, "reasoning_tokens": 14752104, "cache_read_tokens": 2079818560, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 125.8, "avg_query_seconds": 52.4, "median_query_seconds": 29.4, "avg_ttft_seconds": 13.4, "output_time_split": { "reasoning_pct": 85.9, "content_pct": 0.8, "tool_call_pct": 13.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "stepfun/step-5-preview": { "accuracy": 2.857, "latency": 8997.76, "stderr": 2.006, "cost_per_test": 4.250933, "token_totals": { "input_tokens": 4101639238, "output_tokens": 21535694, "reasoning_tokens": 0, "cache_read_tokens": 4065495040, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 217.8, "avg_query_seconds": 23.0, "median_query_seconds": 11.4, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 80.3, "content_pct": 0.4, "tool_call_pct": 19.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "Mini-SWE-agent" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 2.857, "latency": 11504.04, "stderr": 2.006, "cost_per_test": 0.62088, "token_totals": { "input_tokens": 2819218214, "output_tokens": 17831364, "reasoning_tokens": 13255805, "cache_read_tokens": 2777959296, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 165.9, "avg_query_seconds": 50.7, "median_query_seconds": 26.0, "avg_ttft_seconds": 14.4, "output_time_split": { "reasoning_pct": 85.1, "content_pct": 1.4, "tool_call_pct": 13.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "zai/glm-5.3-flash": { "accuracy": 2.857, "latency": 12461.546, "stderr": 2.006, "cost_per_test": 0.524048, "token_totals": { "input_tokens": 1931171934, "output_tokens": 17549106, "reasoning_tokens": 13130932, "cache_read_tokens": 1875697280, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 141.9, "avg_query_seconds": 39.6, "median_query_seconds": 21.5, "avg_ttft_seconds": 6.3, "output_time_split": { "reasoning_pct": 98.9, "content_pct": 1.1, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "Mini-SWE-agent" }, "google/gemini-3.1-pro-preview": { "accuracy": 1.429, "latency": 2638.065, "stderr": 1.429, "cost_per_test": 5.907095, "token_totals": { "input_tokens": 858729252, "output_tokens": 3958785, "reasoning_tokens": 2054169, "cache_read_tokens": 796983036, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 122.2, "avg_query_seconds": 12.0, "median_query_seconds": 7.1, "avg_ttft_seconds": 9.7, "output_time_split": { "reasoning_pct": 84.7, "content_pct": 14.3, "tool_call_pct": 1.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "reflection/Beam-1.0": { "accuracy": 1.429, "latency": 4658.837, "stderr": 1.429, "cost_per_test": null, "token_totals": { "input_tokens": 1817962438, "output_tokens": 4532859, "reasoning_tokens": 2389890, "cache_read_tokens": 1800613696, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": "Mini-SWE-agent" }, "tencent/hy4-preview": { "accuracy": 1.429, "latency": 13679.187, "stderr": 1.429, "cost_per_test": 2.143682, "token_totals": { "input_tokens": 1954683816, "output_tokens": 18902210, "reasoning_tokens": 11136018, "cache_read_tokens": 1928564352, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 174.0, "avg_query_seconds": 31.1, "median_query_seconds": 10.7, "avg_ttft_seconds": 5.5, "output_time_split": { "reasoning_pct": 64.8, "content_pct": 35.2, "tool_call_pct": 0.0 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "Terminus 2" }, "kimi/kimi-k3": { "accuracy": 1.429, "latency": 17349.404, "stderr": 1.429, "cost_per_test": 21.301409, "token_totals": { "input_tokens": 3528346373, "output_tokens": 19628672, "reasoning_tokens": 13477143, "cache_read_tokens": 3477174272, "cache_write_tokens": 49305480 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 213.9, "avg_query_seconds": 57.6, "median_query_seconds": 35.7, "avg_ttft_seconds": 22.3, "output_time_split": { "reasoning_pct": 74.5, "content_pct": 2.4, "tool_call_pct": 23.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 1693.946, "stderr": 0.0, "cost_per_test": 0.17804, "token_totals": { "input_tokens": 256765973, "output_tokens": 3081607, "reasoning_tokens": 2995809, "cache_read_tokens": 228897807, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 76.4, "avg_query_seconds": 2.5, "median_query_seconds": 1.8, "avg_ttft_seconds": 1.6, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 7396.452, "stderr": 0.0, "cost_per_test": 0.581456, "token_totals": { "input_tokens": 1129716462, "output_tokens": 7266560, "reasoning_tokens": 3921207, "cache_read_tokens": 1113668160, "cache_write_tokens": 16009244 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 120.9, "avg_query_seconds": 10.7, "median_query_seconds": 6.9, "avg_ttft_seconds": 5.7, "output_time_split": { "reasoning_pct": 22.5, "content_pct": 1.4, "tool_call_pct": 76.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 10621.573, "stderr": 0.0, "cost_per_test": 0.473154, "token_totals": { "input_tokens": 1765162289, "output_tokens": 17312542, "reasoning_tokens": 12785452, "cache_read_tokens": 1759193848, "cache_write_tokens": 0 }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 136.1, "avg_query_seconds": 10.0, "median_query_seconds": 7.1, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 82.0, "content_pct": 0.4, "tool_call_pct": 17.6 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 14549.775, "stderr": 0.0, "cost_per_test": 11.135637, "token_totals": { "input_tokens": 9837173014, "output_tokens": 30937435, "reasoning_tokens": 20965925, "cache_read_tokens": 9404231680, "cache_write_tokens": null }, "token_tasks_measured": 70, "turn_metrics": { "tasks_measured": 70, "avg_turns": 334.7, "avg_query_seconds": 29.3, "median_query_seconds": 13.5, "avg_ttft_seconds": 16.7, "output_time_split": { "reasoning_pct": 75.6, "content_pct": 2.0, "tool_call_pct": 22.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" } }, "life_sciences": { "openai/gpt-6-astra": { "accuracy": 68.421, "latency": null, "stderr": 10.956, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 43.2, "avg_query_seconds": 104.5, "median_query_seconds": 43.7, "avg_ttft_seconds": 88.5, "output_time_split": { "reasoning_pct": 3.9, "content_pct": 23.3, "tool_call_pct": 72.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5-5": { "accuracy": 52.632, "latency": null, "stderr": 11.769, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 112.1, "avg_query_seconds": 20.8, "median_query_seconds": 14.6, "avg_ttft_seconds": 20.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "openai/gpt-6.1-sol": { "accuracy": 47.368, "latency": null, "stderr": 11.769, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 38.3, "avg_query_seconds": 102.7, "median_query_seconds": 46.6, "avg_ttft_seconds": 84.2, "output_time_split": { "reasoning_pct": 5.1, "content_pct": 16.0, "tool_call_pct": 78.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5-5": { "accuracy": 42.105, "latency": null, "stderr": 11.637, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 232.4, "avg_query_seconds": 18.1, "median_query_seconds": 12.6, "avg_ttft_seconds": 17.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "google/gemini-4-argon": { "accuracy": 42.105, "latency": null, "stderr": 11.637, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 108.9, "avg_query_seconds": 39.3, "median_query_seconds": 33.1, "avg_ttft_seconds": 12.5, "output_time_split": { "reasoning_pct": 55.6, "content_pct": 44.1, "tool_call_pct": 0.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "openai/gpt-6-sol": { "accuracy": 42.105, "latency": null, "stderr": 11.637, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 76.7, "avg_query_seconds": 31.5, "median_query_seconds": 18.1, "avg_ttft_seconds": 14.7, "output_time_split": { "reasoning_pct": 8.8, "content_pct": 4.1, "tool_call_pct": 87.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3.1-pro": { "accuracy": 36.842, "latency": null, "stderr": 11.37, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "meta/muse-wonder-full": { "accuracy": 36.842, "latency": null, "stderr": 11.37, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 167.6, "avg_query_seconds": 64.3, "median_query_seconds": 30.2, "avg_ttft_seconds": 62.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5": { "accuracy": 31.579, "latency": null, "stderr": 10.956, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 163.9, "avg_query_seconds": 18.5, "median_query_seconds": 10.2, "avg_ttft_seconds": 17.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5-1": { "accuracy": 26.316, "latency": null, "stderr": 10.379, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 97.1, "avg_query_seconds": 27.8, "median_query_seconds": 19.5, "avg_ttft_seconds": 26.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "grok/grok-4.6": { "accuracy": 21.053, "latency": null, "stderr": 9.609, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 46.7, "avg_query_seconds": 23.6, "median_query_seconds": 15.2, "avg_ttft_seconds": 4.2, "output_time_split": { "reasoning_pct": 77.5, "content_pct": 22.4, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "meta/muse-wonder": { "accuracy": 21.053, "latency": null, "stderr": 9.609, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 204.2, "avg_query_seconds": 13.7, "median_query_seconds": 8.5, "avg_ttft_seconds": 12.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-sol": { "accuracy": 21.053, "latency": null, "stderr": 9.609, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 56.6, "avg_query_seconds": 19.9, "median_query_seconds": 14.4, "avg_ttft_seconds": 10.3, "output_time_split": { "reasoning_pct": 15.1, "content_pct": 33.6, "tool_call_pct": 51.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-haiku-5-5": { "accuracy": 15.789, "latency": null, "stderr": 8.595, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 189.5, "avg_query_seconds": 8.8, "median_query_seconds": 4.7, "avg_ttft_seconds": 7.3, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3": { "accuracy": 15.789, "latency": null, "stderr": 8.595, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 98.3, "avg_query_seconds": 8.6, "median_query_seconds": 5.8, "avg_ttft_seconds": 4.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3_max": { "accuracy": 15.789, "latency": null, "stderr": 8.595, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 118.2, "avg_query_seconds": 11.2, "median_query_seconds": 7.7, "avg_ttft_seconds": 5.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-terra": { "accuracy": 15.789, "latency": null, "stderr": 8.595, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 75.6, "avg_query_seconds": 19.7, "median_query_seconds": 14.7, "avg_ttft_seconds": 8.9, "output_time_split": { "reasoning_pct": 26.4, "content_pct": 8.7, "tool_call_pct": 64.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "zai/glm-5.3": { "accuracy": 15.789, "latency": null, "stderr": 8.595, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 200.4, "avg_query_seconds": 23.6, "median_query_seconds": 13.5, "avg_ttft_seconds": 6.6, "output_time_split": { "reasoning_pct": 99.1, "content_pct": 0.9, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-max": { "accuracy": 10.526, "latency": null, "stderr": 7.234, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 153.2, "avg_query_seconds": 47.8, "median_query_seconds": 33.4, "avg_ttft_seconds": 10.3, "output_time_split": { "reasoning_pct": 76.6, "content_pct": 0.3, "tool_call_pct": 23.0 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5": { "accuracy": 10.526, "latency": null, "stderr": 7.234, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 133.2, "avg_query_seconds": 22.4, "median_query_seconds": 9.2, "avg_ttft_seconds": 19.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5": { "accuracy": 10.526, "latency": null, "stderr": 7.234, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 247.1, "avg_query_seconds": 16.2, "median_query_seconds": 7.8, "avg_ttft_seconds": 15.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "google/gemini-3.6-flash": { "accuracy": 10.526, "latency": null, "stderr": 7.234, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 77.1, "avg_query_seconds": 6.3, "median_query_seconds": 4.9, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 81.5, "content_pct": 17.8, "tool_call_pct": 0.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.7-flash": { "accuracy": 10.526, "latency": null, "stderr": 7.234, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 87.2, "avg_query_seconds": 5.2, "median_query_seconds": 4.2, "avg_ttft_seconds": 3.7, "output_time_split": { "reasoning_pct": 94.7, "content_pct": 3.7, "tool_call_pct": 1.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "openai/gpt-6-luna": { "accuracy": 10.526, "latency": null, "stderr": 7.234, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 61.6, "avg_query_seconds": 22.3, "median_query_seconds": 12.7, "avg_ttft_seconds": 7.5, "output_time_split": { "reasoning_pct": 51.2, "content_pct": 7.3, "tool_call_pct": 41.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 10.526, "latency": null, "stderr": 7.234, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 115.5, "avg_query_seconds": 54.6, "median_query_seconds": 31.8, "avg_ttft_seconds": 16.6, "output_time_split": { "reasoning_pct": 84.9, "content_pct": 1.6, "tool_call_pct": 13.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 5.263, "latency": null, "stderr": 5.263, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 270.8, "avg_query_seconds": 8.7, "median_query_seconds": 6.4, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 64.4, "content_pct": 0.7, "tool_call_pct": 34.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "fireworks/ember-1": { "accuracy": 5.263, "latency": null, "stderr": 5.263, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 120.4, "avg_query_seconds": 29.3, "median_query_seconds": 19.7, "avg_ttft_seconds": 7.2, "output_time_split": { "reasoning_pct": 72.4, "content_pct": 1.5, "tool_call_pct": 26.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "google/gemini-3.5-flash": { "accuracy": 5.263, "latency": null, "stderr": 5.263, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 69.9, "avg_query_seconds": 6.4, "median_query_seconds": 4.0, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 64.5, "content_pct": 34.8, "tool_call_pct": 0.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.8-flash": { "accuracy": 5.263, "latency": null, "stderr": 5.263, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 133.9, "avg_query_seconds": 5.1, "median_query_seconds": 4.0, "avg_ttft_seconds": 2.9, "output_time_split": { "reasoning_pct": 98.7, "content_pct": 0.0, "tool_call_pct": 1.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "grok/grok-4.7": { "accuracy": 5.263, "latency": null, "stderr": 5.263, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 76.2, "avg_query_seconds": 23.7, "median_query_seconds": 18.6, "avg_ttft_seconds": 2.8, "output_time_split": { "reasoning_pct": 94.2, "content_pct": 5.8, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 5.263, "latency": null, "stderr": 5.263, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 98.0, "avg_query_seconds": 49.4, "median_query_seconds": 33.4, "avg_ttft_seconds": 10.9, "output_time_split": { "reasoning_pct": 83.7, "content_pct": 1.1, "tool_call_pct": 15.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "zai/glm-5.3-flash": { "accuracy": 5.263, "latency": null, "stderr": 5.263, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 125.0, "avg_query_seconds": 31.9, "median_query_seconds": 20.1, "avg_ttft_seconds": 5.4, "output_time_split": { "reasoning_pct": 99.4, "content_pct": 0.6, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 168.0, "avg_query_seconds": 19.9, "median_query_seconds": 11.0, "avg_ttft_seconds": 10.7, "output_time_split": { "reasoning_pct": 72.6, "content_pct": 2.2, "tool_call_pct": 25.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 87.2, "avg_query_seconds": 35.5, "median_query_seconds": 22.0, "avg_ttft_seconds": 28.8, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 167.5, "avg_query_seconds": 7.2, "median_query_seconds": 5.1, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 81.4, "content_pct": 0.5, "tool_call_pct": 18.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 102.5, "avg_query_seconds": 9.5, "median_query_seconds": 7.2, "avg_ttft_seconds": 2.0, "output_time_split": { "reasoning_pct": 82.0, "content_pct": 0.6, "tool_call_pct": 17.4 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 74.5, "avg_query_seconds": 13.7, "median_query_seconds": 6.5, "avg_ttft_seconds": 11.1, "output_time_split": { "reasoning_pct": 77.2, "content_pct": 22.1, "tool_call_pct": 0.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 57.4, "avg_query_seconds": 2.2, "median_query_seconds": 1.5, "avg_ttft_seconds": 1.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 139.3, "avg_query_seconds": 46.2, "median_query_seconds": 30.2, "avg_ttft_seconds": 15.3, "output_time_split": { "reasoning_pct": 73.9, "content_pct": 2.5, "tool_call_pct": 23.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 106.1, "avg_query_seconds": 11.4, "median_query_seconds": 7.0, "avg_ttft_seconds": 5.1, "output_time_split": { "reasoning_pct": 18.1, "content_pct": 1.2, "tool_call_pct": 80.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": "Mini-SWE-agent" }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 175.2, "avg_query_seconds": 22.9, "median_query_seconds": 12.8, "avg_ttft_seconds": 4.5, "output_time_split": { "reasoning_pct": 79.5, "content_pct": 0.5, "tool_call_pct": 20.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "Mini-SWE-agent" }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 19, "avg_turns": 114.1, "avg_query_seconds": 25.8, "median_query_seconds": 12.2, "avg_ttft_seconds": 4.4, "output_time_split": { "reasoning_pct": 56.0, "content_pct": 44.0, "tool_call_pct": 0.0 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "Terminus 2" } }, "physical_sciences": { "anthropic/claude-sonnet-5-5": { "accuracy": 58.824, "latency": null, "stderr": 12.304, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 290.2, "avg_query_seconds": 16.6, "median_query_seconds": 10.2, "avg_ttft_seconds": 16.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5-5": { "accuracy": 47.059, "latency": null, "stderr": 12.478, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 123.4, "avg_query_seconds": 24.2, "median_query_seconds": 14.4, "avg_ttft_seconds": 23.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3.1-pro": { "accuracy": 47.059, "latency": null, "stderr": 12.478, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "openai/gpt-6-astra": { "accuracy": 47.059, "latency": null, "stderr": 12.478, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 46.8, "avg_query_seconds": 153.3, "median_query_seconds": 80.2, "avg_ttft_seconds": 130.2, "output_time_split": { "reasoning_pct": 2.5, "content_pct": 7.3, "tool_call_pct": 90.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "google/gemini-4-argon": { "accuracy": 41.176, "latency": null, "stderr": 12.304, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 105.9, "avg_query_seconds": 45.0, "median_query_seconds": 32.8, "avg_ttft_seconds": 13.5, "output_time_split": { "reasoning_pct": 53.0, "content_pct": 46.7, "tool_call_pct": 0.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "openai/gpt-6.1-sol": { "accuracy": 41.176, "latency": null, "stderr": 12.304, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 39.3, "avg_query_seconds": 141.7, "median_query_seconds": 67.6, "avg_ttft_seconds": 116.6, "output_time_split": { "reasoning_pct": 3.2, "content_pct": 9.9, "tool_call_pct": 86.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5-1": { "accuracy": 35.294, "latency": null, "stderr": 11.947, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 106.0, "avg_query_seconds": 29.1, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5": { "accuracy": 35.294, "latency": null, "stderr": 11.947, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 217.3, "avg_query_seconds": 20.1, "median_query_seconds": 8.1, "avg_ttft_seconds": 18.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "meta/muse-wonder-full": { "accuracy": 35.294, "latency": null, "stderr": 11.947, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 191.9, "avg_query_seconds": 48.9, "median_query_seconds": 16.3, "avg_ttft_seconds": 47.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5": { "accuracy": 29.412, "latency": null, "stderr": 11.391, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 189.6, "avg_query_seconds": 24.0, "median_query_seconds": 9.0, "avg_ttft_seconds": 21.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-sol": { "accuracy": 29.412, "latency": null, "stderr": 11.391, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 61.6, "avg_query_seconds": 20.3, "median_query_seconds": 13.1, "avg_ttft_seconds": 9.6, "output_time_split": { "reasoning_pct": 12.4, "content_pct": 17.9, "tool_call_pct": 69.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-max": { "accuracy": 23.529, "latency": null, "stderr": 10.605, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 213.2, "avg_query_seconds": 50.3, "median_query_seconds": 31.0, "avg_ttft_seconds": 10.8, "output_time_split": { "reasoning_pct": 78.9, "content_pct": 0.4, "tool_call_pct": 20.8 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "grok/grok-4.7": { "accuracy": 23.529, "latency": null, "stderr": 10.605, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 93.9, "avg_query_seconds": 29.4, "median_query_seconds": 18.8, "avg_ttft_seconds": 4.4, "output_time_split": { "reasoning_pct": 94.1, "content_pct": 5.9, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "meta/muse-wonder": { "accuracy": 23.529, "latency": null, "stderr": 10.605, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 302.2, "avg_query_seconds": 10.9, "median_query_seconds": 5.6, "avg_ttft_seconds": 9.8, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-6-sol": { "accuracy": 23.529, "latency": null, "stderr": 10.605, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 93.6, "avg_query_seconds": 25.1, "median_query_seconds": 15.4, "avg_ttft_seconds": 13.8, "output_time_split": { "reasoning_pct": 13.9, "content_pct": 7.0, "tool_call_pct": 79.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "google/gemini-3.7-flash": { "accuracy": 17.647, "latency": null, "stderr": 9.531, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 120.6, "avg_query_seconds": 6.5, "median_query_seconds": 5.0, "avg_ttft_seconds": 4.5, "output_time_split": { "reasoning_pct": 98.4, "content_pct": 0.0, "tool_call_pct": 1.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "anthropic/claude-haiku-5-5": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 256.4, "avg_query_seconds": 8.1, "median_query_seconds": 3.8, "avg_ttft_seconds": 6.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-4-8": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 175.6, "avg_query_seconds": 29.2, "median_query_seconds": 13.9, "avg_ttft_seconds": 24.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 480.4, "avg_query_seconds": 13.1, "median_query_seconds": 7.1, "avg_ttft_seconds": 12.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 327.8, "avg_query_seconds": 6.6, "median_query_seconds": 4.5, "avg_ttft_seconds": 2.9, "output_time_split": { "reasoning_pct": 81.9, "content_pct": 0.4, "tool_call_pct": 17.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "google/gemini-3.8-flash": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 160.7, "avg_query_seconds": 5.4, "median_query_seconds": 4.1, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 98.6, "content_pct": 0.0, "tool_call_pct": 1.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "grok/grok-4.6": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 64.9, "avg_query_seconds": 24.9, "median_query_seconds": 13.3, "avg_ttft_seconds": 4.6, "output_time_split": { "reasoning_pct": 77.1, "content_pct": 22.8, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3_max": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 144.1, "avg_query_seconds": 13.9, "median_query_seconds": 9.8, "avg_ttft_seconds": 6.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-terra": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 89.6, "avg_query_seconds": 19.1, "median_query_seconds": 12.8, "avg_ttft_seconds": 8.9, "output_time_split": { "reasoning_pct": 23.0, "content_pct": 8.8, "tool_call_pct": 68.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "stepfun/step-5-preview": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 268.6, "avg_query_seconds": 21.1, "median_query_seconds": 9.7, "avg_ttft_seconds": 5.3, "output_time_split": { "reasoning_pct": 80.4, "content_pct": 0.3, "tool_call_pct": 19.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 499.7, "avg_query_seconds": 8.1, "median_query_seconds": 5.3, "avg_ttft_seconds": 3.2, "output_time_split": { "reasoning_pct": 68.1, "content_pct": 0.4, "tool_call_pct": 31.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "fireworks/ember-1": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 403.5, "avg_query_seconds": 18.6, "median_query_seconds": 10.0, "avg_ttft_seconds": 5.9, "output_time_split": { "reasoning_pct": 69.7, "content_pct": 1.9, "tool_call_pct": 28.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "google/gemini-3.1-pro-preview": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 228.2, "avg_query_seconds": 11.1, "median_query_seconds": 6.9, "avg_ttft_seconds": 9.5, "output_time_split": { "reasoning_pct": 89.6, "content_pct": 8.4, "tool_call_pct": 2.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.5-flash": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 95.3, "avg_query_seconds": 7.4, "median_query_seconds": 4.6, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 53.8, "content_pct": 45.5, "tool_call_pct": 0.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "tencent/hy4-preview": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 178.9, "avg_query_seconds": 26.0, "median_query_seconds": 9.9, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 57.4, "content_pct": 42.6, "tool_call_pct": 0.0 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "Terminus 2" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 129.1, "avg_query_seconds": 55.0, "median_query_seconds": 29.2, "avg_ttft_seconds": 14.6, "output_time_split": { "reasoning_pct": 86.7, "content_pct": 0.8, "tool_call_pct": 12.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "zai/glm-5.3": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 236.0, "avg_query_seconds": 27.0, "median_query_seconds": 12.6, "avg_ttft_seconds": 7.0, "output_time_split": { "reasoning_pct": 99.3, "content_pct": 0.7, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "zai/glm-5.3-flash": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 153.3, "avg_query_seconds": 39.4, "median_query_seconds": 21.6, "avg_ttft_seconds": 6.5, "output_time_split": { "reasoning_pct": 99.1, "content_pct": 0.9, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 476.3, "avg_query_seconds": 28.2, "median_query_seconds": 13.2, "avg_ttft_seconds": 16.3, "output_time_split": { "reasoning_pct": 76.8, "content_pct": 2.0, "tool_call_pct": 21.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 156.4, "avg_query_seconds": 10.1, "median_query_seconds": 7.2, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 82.1, "content_pct": 0.5, "tool_call_pct": 17.4 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 16, "avg_turns": 119.9, "avg_query_seconds": 7.8, "median_query_seconds": 5.3, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 91.7, "content_pct": 7.6, "tool_call_pct": 0.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 78.3, "avg_query_seconds": 2.5, "median_query_seconds": 1.7, "avg_ttft_seconds": 1.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 264.1, "avg_query_seconds": 52.3, "median_query_seconds": 30.4, "avg_ttft_seconds": 20.3, "output_time_split": { "reasoning_pct": 74.9, "content_pct": 2.6, "tool_call_pct": 22.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 133.8, "avg_query_seconds": 9.5, "median_query_seconds": 6.1, "avg_ttft_seconds": 5.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 127.2, "avg_query_seconds": 9.9, "median_query_seconds": 6.4, "avg_ttft_seconds": 5.5, "output_time_split": { "reasoning_pct": 22.2, "content_pct": 1.5, "tool_call_pct": 76.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 89.8, "avg_query_seconds": 19.1, "median_query_seconds": 10.0, "avg_ttft_seconds": 7.5, "output_time_split": { "reasoning_pct": 47.5, "content_pct": 5.2, "tool_call_pct": 47.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": "Mini-SWE-agent" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 202.1, "avg_query_seconds": 46.2, "median_query_seconds": 23.0, "avg_ttft_seconds": 13.9, "output_time_split": { "reasoning_pct": 85.1, "content_pct": 1.4, "tool_call_pct": 13.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" } }, "earth_sciences": { "google/gemini-4-argon": { "accuracy": 62.5, "latency": null, "stderr": 18.298, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 137.0, "avg_query_seconds": 39.7, "median_query_seconds": 30.1, "avg_ttft_seconds": 14.3, "output_time_split": { "reasoning_pct": 42.6, "content_pct": 57.0, "tool_call_pct": 0.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "meta/muse-wonder-full": { "accuracy": 62.5, "latency": null, "stderr": 18.298, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 222.1, "avg_query_seconds": 50.7, "median_query_seconds": 18.6, "avg_ttft_seconds": 48.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-6-astra": { "accuracy": 62.5, "latency": null, "stderr": 18.298, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 44.8, "avg_query_seconds": 106.7, "median_query_seconds": 67.0, "avg_ttft_seconds": 87.7, "output_time_split": { "reasoning_pct": 4.0, "content_pct": 26.2, "tool_call_pct": 69.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6.1-sol": { "accuracy": 62.5, "latency": null, "stderr": 18.298, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 46.8, "avg_query_seconds": 139.4, "median_query_seconds": 66.1, "avg_ttft_seconds": 121.8, "output_time_split": { "reasoning_pct": 4.0, "content_pct": 11.8, "tool_call_pct": 84.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "meta/muse-wonder": { "accuracy": 50.0, "latency": null, "stderr": 18.898, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 252.0, "avg_query_seconds": 11.1, "median_query_seconds": 6.6, "avg_ttft_seconds": 10.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5-5": { "accuracy": 37.5, "latency": null, "stderr": 18.298, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 126.2, "avg_query_seconds": 18.6, "median_query_seconds": 11.8, "avg_ttft_seconds": 17.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5": { "accuracy": 25.0, "latency": null, "stderr": 16.366, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 166.0, "avg_query_seconds": 20.5, "median_query_seconds": 10.5, "avg_ttft_seconds": 17.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5-1": { "accuracy": 25.0, "latency": null, "stderr": 16.366, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 119.0, "avg_query_seconds": 21.8, "median_query_seconds": 14.2, "avg_ttft_seconds": 20.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "google/gemini-3.5-flash": { "accuracy": 25.0, "latency": null, "stderr": 16.366, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 102.1, "avg_query_seconds": 9.0, "median_query_seconds": 5.1, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 48.4, "content_pct": 51.0, "tool_call_pct": 0.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "grok/grok-4.7": { "accuracy": 25.0, "latency": null, "stderr": 16.366, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 99.6, "avg_query_seconds": 21.6, "median_query_seconds": 14.8, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 95.7, "content_pct": 4.3, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-sol": { "accuracy": 25.0, "latency": null, "stderr": 16.366, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 83.2, "avg_query_seconds": 16.4, "median_query_seconds": 11.9, "avg_ttft_seconds": 8.3, "output_time_split": { "reasoning_pct": 20.3, "content_pct": 29.1, "tool_call_pct": 50.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6-sol": { "accuracy": 25.0, "latency": null, "stderr": 16.366, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 129.5, "avg_query_seconds": 21.3, "median_query_seconds": 13.3, "avg_ttft_seconds": 11.4, "output_time_split": { "reasoning_pct": 17.5, "content_pct": 5.0, "tool_call_pct": 77.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-max": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 198.8, "avg_query_seconds": 42.4, "median_query_seconds": 29.1, "avg_ttft_seconds": 11.2, "output_time_split": { "reasoning_pct": 72.7, "content_pct": 0.3, "tool_call_pct": 27.0 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 246.1, "avg_query_seconds": 16.8, "median_query_seconds": 9.1, "avg_ttft_seconds": 16.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5-5": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 313.9, "avg_query_seconds": 13.9, "median_query_seconds": 9.3, "avg_ttft_seconds": 13.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "fireworks/ember-1": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 245.9, "avg_query_seconds": 22.2, "median_query_seconds": 15.3, "avg_ttft_seconds": 6.8, "output_time_split": { "reasoning_pct": 64.4, "content_pct": 1.7, "tool_call_pct": 33.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "google/gemini-3.6-flash": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 103.1, "avg_query_seconds": 8.8, "median_query_seconds": 6.3, "avg_ttft_seconds": 3.2, "output_time_split": { "reasoning_pct": 99.5, "content_pct": 0.0, "tool_call_pct": 0.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "grok/grok-4.6": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 68.0, "avg_query_seconds": 20.8, "median_query_seconds": 11.9, "avg_ttft_seconds": 5.3, "output_time_split": { "reasoning_pct": 76.8, "content_pct": 23.2, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 235.0, "avg_query_seconds": 52.4, "median_query_seconds": 31.3, "avg_ttft_seconds": 20.8, "output_time_split": { "reasoning_pct": 72.9, "content_pct": 2.6, "tool_call_pct": 24.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3.1-pro": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "reflection/Beam-1.0": { "accuracy": 12.5, "latency": null, "stderr": 12.5, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 408.1, "avg_query_seconds": 23.1, "median_query_seconds": 12.1, "avg_ttft_seconds": 14.0, "output_time_split": { "reasoning_pct": 69.9, "content_pct": 1.7, "tool_call_pct": 28.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 174.8, "avg_query_seconds": 8.3, "median_query_seconds": 4.1, "avg_ttft_seconds": 6.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 108.8, "avg_query_seconds": 32.1, "median_query_seconds": 18.2, "avg_ttft_seconds": 25.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 275.1, "avg_query_seconds": 16.8, "median_query_seconds": 8.5, "avg_ttft_seconds": 15.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 255.6, "avg_query_seconds": 6.7, "median_query_seconds": 4.7, "avg_ttft_seconds": 2.6, "output_time_split": { "reasoning_pct": 79.5, "content_pct": 0.4, "tool_call_pct": 20.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 286.4, "avg_query_seconds": 9.1, "median_query_seconds": 6.4, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 65.5, "content_pct": 0.4, "tool_call_pct": 34.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 123.9, "avg_query_seconds": 8.4, "median_query_seconds": 5.9, "avg_ttft_seconds": 2.0, "output_time_split": { "reasoning_pct": 79.8, "content_pct": 0.3, "tool_call_pct": 19.9 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 152.2, "avg_query_seconds": 9.5, "median_query_seconds": 6.7, "avg_ttft_seconds": 7.7, "output_time_split": { "reasoning_pct": 96.3, "content_pct": 2.6, "tool_call_pct": 1.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 104.1, "avg_query_seconds": 6.7, "median_query_seconds": 5.1, "avg_ttft_seconds": 4.3, "output_time_split": { "reasoning_pct": 98.5, "content_pct": 0.0, "tool_call_pct": 1.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 136.2, "avg_query_seconds": 5.4, "median_query_seconds": 4.1, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 98.0, "content_pct": 0.5, "tool_call_pct": 1.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 95.4, "avg_query_seconds": 2.2, "median_query_seconds": 1.9, "avg_ttft_seconds": 1.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 147.1, "avg_query_seconds": 8.7, "median_query_seconds": 5.6, "avg_ttft_seconds": 5.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 147.4, "avg_query_seconds": 13.6, "median_query_seconds": 9.4, "avg_ttft_seconds": 6.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 131.1, "avg_query_seconds": 8.9, "median_query_seconds": 6.2, "avg_ttft_seconds": 4.9, "output_time_split": { "reasoning_pct": 31.4, "content_pct": 2.2, "tool_call_pct": 66.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 90.5, "avg_query_seconds": 17.1, "median_query_seconds": 10.4, "avg_ttft_seconds": 7.3, "output_time_split": { "reasoning_pct": 28.5, "content_pct": 10.4, "tool_call_pct": 61.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 70.5, "avg_query_seconds": 20.5, "median_query_seconds": 13.9, "avg_ttft_seconds": 6.7, "output_time_split": { "reasoning_pct": 57.3, "content_pct": 5.6, "tool_call_pct": 37.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 253.9, "avg_query_seconds": 19.3, "median_query_seconds": 11.5, "avg_ttft_seconds": 4.8, "output_time_split": { "reasoning_pct": 75.9, "content_pct": 0.3, "tool_call_pct": 23.8 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "Mini-SWE-agent" }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 146.4, "avg_query_seconds": 26.8, "median_query_seconds": 11.9, "avg_ttft_seconds": 4.5, "output_time_split": { "reasoning_pct": 55.8, "content_pct": 44.2, "tool_call_pct": 0.0 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "Terminus 2" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 161.4, "avg_query_seconds": 42.1, "median_query_seconds": 27.4, "avg_ttft_seconds": 11.8, "output_time_split": { "reasoning_pct": 83.5, "content_pct": 0.8, "tool_call_pct": 15.8 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 239.2, "avg_query_seconds": 39.1, "median_query_seconds": 22.0, "avg_ttft_seconds": 12.8, "output_time_split": { "reasoning_pct": 83.0, "content_pct": 1.5, "tool_call_pct": 15.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "zai/glm-5.3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 212.5, "avg_query_seconds": 22.5, "median_query_seconds": 12.2, "avg_ttft_seconds": 6.6, "output_time_split": { "reasoning_pct": 99.4, "content_pct": 0.6, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 150.6, "avg_query_seconds": 32.9, "median_query_seconds": 19.6, "avg_ttft_seconds": 6.2, "output_time_split": { "reasoning_pct": 98.7, "content_pct": 1.2, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "Mini-SWE-agent" } }, "mathematical_sciences": { "openai/gpt-6-astra": { "accuracy": 88.235, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 51.9, "avg_query_seconds": 88.0, "median_query_seconds": 52.5, "avg_ttft_seconds": 66.0, "output_time_split": { "reasoning_pct": 2.7, "content_pct": 24.0, "tool_call_pct": 73.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6.1-sol": { "accuracy": 76.471, "latency": null, "stderr": 10.605, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 44.6, "avg_query_seconds": 121.0, "median_query_seconds": 58.8, "avg_ttft_seconds": 93.9, "output_time_split": { "reasoning_pct": 4.6, "content_pct": 10.1, "tool_call_pct": 85.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5-1": { "accuracy": 64.706, "latency": null, "stderr": 11.947, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 96.5, "avg_query_seconds": 32.3, "median_query_seconds": 18.1, "avg_ttft_seconds": 30.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5-5": { "accuracy": 58.824, "latency": null, "stderr": 12.304, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 102.3, "avg_query_seconds": 26.1, "median_query_seconds": 14.1, "avg_ttft_seconds": 24.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5-5": { "accuracy": 58.824, "latency": null, "stderr": 12.304, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 287.2, "avg_query_seconds": 16.1, "median_query_seconds": 9.8, "avg_ttft_seconds": 15.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "google/gemini-4-argon": { "accuracy": 58.824, "latency": null, "stderr": 12.304, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 110.2, "avg_query_seconds": 58.6, "median_query_seconds": 41.5, "avg_ttft_seconds": 16.4, "output_time_split": { "reasoning_pct": 54.0, "content_pct": 45.8, "tool_call_pct": 0.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "meta/muse-wonder-full": { "accuracy": 41.176, "latency": null, "stderr": 12.304, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 207.1, "avg_query_seconds": 56.4, "median_query_seconds": 16.5, "avg_ttft_seconds": 54.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5": { "accuracy": 29.412, "latency": null, "stderr": 11.391, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 176.4, "avg_query_seconds": 21.5, "median_query_seconds": 8.5, "avg_ttft_seconds": 19.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "openai/gpt-6-sol": { "accuracy": 29.412, "latency": null, "stderr": 11.391, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 125.8, "avg_query_seconds": 23.0, "median_query_seconds": 13.4, "avg_ttft_seconds": 13.1, "output_time_split": { "reasoning_pct": 15.2, "content_pct": 5.0, "tool_call_pct": 79.8 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3.1-pro": { "accuracy": 23.529, "latency": null, "stderr": 10.605, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "meta/muse-wonder": { "accuracy": 23.529, "latency": null, "stderr": 10.605, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 213.8, "avg_query_seconds": 16.9, "median_query_seconds": 7.9, "avg_ttft_seconds": 15.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-max": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 177.4, "avg_query_seconds": 48.5, "median_query_seconds": 28.4, "avg_ttft_seconds": 11.1, "output_time_split": { "reasoning_pct": 76.7, "content_pct": 0.4, "tool_call_pct": 22.9 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 177.5, "avg_query_seconds": 25.4, "median_query_seconds": 8.8, "avg_ttft_seconds": 23.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-haiku-5-5": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 226.0, "avg_query_seconds": 9.3, "median_query_seconds": 3.7, "avg_ttft_seconds": 7.9, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "google/gemini-3.8-flash": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 149.1, "avg_query_seconds": 7.0, "median_query_seconds": 4.6, "avg_ttft_seconds": 3.8, "output_time_split": { "reasoning_pct": 96.3, "content_pct": 2.6, "tool_call_pct": 1.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3_max": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 131.5, "avg_query_seconds": 14.1, "median_query_seconds": 8.1, "avg_ttft_seconds": 6.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-sol": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 93.9, "avg_query_seconds": 21.1, "median_query_seconds": 12.7, "avg_ttft_seconds": 11.1, "output_time_split": { "reasoning_pct": 14.1, "content_pct": 29.8, "tool_call_pct": 56.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-terra": { "accuracy": 11.765, "latency": null, "stderr": 8.055, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 128.4, "avg_query_seconds": 19.8, "median_query_seconds": 11.1, "avg_ttft_seconds": 10.2, "output_time_split": { "reasoning_pct": 24.3, "content_pct": 9.6, "tool_call_pct": 66.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-4-8": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 101.9, "avg_query_seconds": 46.0, "median_query_seconds": 26.3, "avg_ttft_seconds": 38.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 206.8, "avg_query_seconds": 8.4, "median_query_seconds": 5.0, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 82.9, "content_pct": 0.5, "tool_call_pct": 16.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 367.6, "avg_query_seconds": 9.1, "median_query_seconds": 5.5, "avg_ttft_seconds": 2.9, "output_time_split": { "reasoning_pct": 69.8, "content_pct": 0.5, "tool_call_pct": 29.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "fireworks/ember-1": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 308.3, "avg_query_seconds": 26.9, "median_query_seconds": 14.2, "avg_ttft_seconds": 6.5, "output_time_split": { "reasoning_pct": 75.6, "content_pct": 1.7, "tool_call_pct": 22.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 5.882, "latency": null, "stderr": 5.882, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 136.3, "avg_query_seconds": 57.3, "median_query_seconds": 26.8, "avg_ttft_seconds": 15.1, "output_time_split": { "reasoning_pct": 88.3, "content_pct": 0.6, "tool_call_pct": 11.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 351.8, "avg_query_seconds": 33.0, "median_query_seconds": 14.4, "avg_ttft_seconds": 17.9, "output_time_split": { "reasoning_pct": 77.6, "content_pct": 2.2, "tool_call_pct": 20.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 334.2, "avg_query_seconds": 19.3, "median_query_seconds": 8.4, "avg_ttft_seconds": 18.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 141.3, "avg_query_seconds": 10.8, "median_query_seconds": 7.1, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 84.6, "content_pct": 0.4, "tool_call_pct": 15.0 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 81.0, "avg_query_seconds": 14.6, "median_query_seconds": 8.4, "avg_ttft_seconds": 10.7, "output_time_split": { "reasoning_pct": 86.3, "content_pct": 13.3, "tool_call_pct": 0.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 88.4, "avg_query_seconds": 10.6, "median_query_seconds": 5.7, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 61.7, "content_pct": 37.8, "tool_call_pct": 0.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 113.4, "avg_query_seconds": 13.3, "median_query_seconds": 9.8, "avg_ttft_seconds": 3.9, "output_time_split": { "reasoning_pct": 87.6, "content_pct": 11.8, "tool_call_pct": 0.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 130.1, "avg_query_seconds": 8.0, "median_query_seconds": 5.9, "avg_ttft_seconds": 4.7, "output_time_split": { "reasoning_pct": 98.6, "content_pct": 0.0, "tool_call_pct": 1.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "grok/grok-4.6": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 93.6, "avg_query_seconds": 22.5, "median_query_seconds": 11.8, "avg_ttft_seconds": 5.0, "output_time_split": { "reasoning_pct": 83.1, "content_pct": 16.9, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "grok/grok-4.7": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 116.4, "avg_query_seconds": 27.4, "median_query_seconds": 17.1, "avg_ttft_seconds": 4.3, "output_time_split": { "reasoning_pct": 95.2, "content_pct": 4.8, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 100.6, "avg_query_seconds": 2.7, "median_query_seconds": 2.0, "avg_ttft_seconds": 1.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 220.8, "avg_query_seconds": 69.5, "median_query_seconds": 45.8, "avg_ttft_seconds": 28.9, "output_time_split": { "reasoning_pct": 77.7, "content_pct": 2.1, "tool_call_pct": 20.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 133.1, "avg_query_seconds": 10.4, "median_query_seconds": 7.0, "avg_ttft_seconds": 5.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 124.1, "avg_query_seconds": 11.8, "median_query_seconds": 7.5, "avg_ttft_seconds": 7.0, "output_time_split": { "reasoning_pct": 23.7, "content_pct": 1.2, "tool_call_pct": 75.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 91.3, "avg_query_seconds": 18.5, "median_query_seconds": 8.2, "avg_ttft_seconds": 7.0, "output_time_split": { "reasoning_pct": 44.9, "content_pct": 4.0, "tool_call_pct": 51.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": "Mini-SWE-agent" }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 220.5, "avg_query_seconds": 25.8, "median_query_seconds": 11.4, "avg_ttft_seconds": 5.9, "output_time_split": { "reasoning_pct": 82.9, "content_pct": 0.4, "tool_call_pct": 16.7 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "Mini-SWE-agent" }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 257.5, "avg_query_seconds": 37.0, "median_query_seconds": 10.5, "avg_ttft_seconds": 6.6, "output_time_split": { "reasoning_pct": 73.9, "content_pct": 26.1, "tool_call_pct": 0.0 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "Terminus 2" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 168.5, "avg_query_seconds": 57.1, "median_query_seconds": 26.5, "avg_ttft_seconds": 15.8, "output_time_split": { "reasoning_pct": 86.5, "content_pct": 1.3, "tool_call_pct": 12.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "zai/glm-5.3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 217.7, "avg_query_seconds": 27.7, "median_query_seconds": 11.3, "avg_ttft_seconds": 7.3, "output_time_split": { "reasoning_pct": 98.7, "content_pct": 1.3, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 17, "avg_turns": 132.6, "avg_query_seconds": 48.9, "median_query_seconds": 21.2, "avg_ttft_seconds": 6.5, "output_time_split": { "reasoning_pct": 98.6, "content_pct": 1.3, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "Mini-SWE-agent" } }, "engineering_sciences": { "anthropic/claude-fable-5-1": { "accuracy": 44.444, "latency": null, "stderr": 17.568, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 129.3, "avg_query_seconds": 31.4, "median_query_seconds": 20.0, "avg_ttft_seconds": 29.8, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5-5": { "accuracy": 44.444, "latency": null, "stderr": 17.568, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 357.1, "avg_query_seconds": 17.5, "median_query_seconds": 11.4, "avg_ttft_seconds": 17.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "openai/gpt-6-astra": { "accuracy": 33.333, "latency": null, "stderr": 16.667, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 48.9, "avg_query_seconds": 164.7, "median_query_seconds": 103.5, "avg_ttft_seconds": 134.7, "output_time_split": { "reasoning_pct": 1.7, "content_pct": 12.3, "tool_call_pct": 86.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6.1-sol": { "accuracy": 33.333, "latency": null, "stderr": 16.667, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 46.6, "avg_query_seconds": 168.0, "median_query_seconds": 89.2, "avg_ttft_seconds": 138.0, "output_time_split": { "reasoning_pct": 3.0, "content_pct": 7.7, "tool_call_pct": 89.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5-5": { "accuracy": 22.222, "latency": null, "stderr": 14.699, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 136.9, "avg_query_seconds": 23.4, "median_query_seconds": 15.4, "avg_ttft_seconds": 22.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3.1-pro": { "accuracy": 22.222, "latency": null, "stderr": 14.699, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "openai/gpt-6-sol": { "accuracy": 22.222, "latency": null, "stderr": 14.699, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 129.7, "avg_query_seconds": 30.8, "median_query_seconds": 20.1, "avg_ttft_seconds": 15.4, "output_time_split": { "reasoning_pct": 12.6, "content_pct": 8.7, "tool_call_pct": 78.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-5": { "accuracy": 11.111, "latency": null, "stderr": 11.111, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 189.8, "avg_query_seconds": 25.3, "median_query_seconds": 13.7, "avg_ttft_seconds": 24.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "google/gemini-3.8-flash": { "accuracy": 11.111, "latency": null, "stderr": 11.111, "cost_per_test": null, "turn_metrics": { "tasks_measured": 8, "avg_turns": 149.4, "avg_query_seconds": 5.9, "median_query_seconds": 4.7, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 98.8, "content_pct": 0.0, "tool_call_pct": 1.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-4-argon": { "accuracy": 11.111, "latency": null, "stderr": 11.111, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 119.8, "avg_query_seconds": 54.1, "median_query_seconds": 41.7, "avg_ttft_seconds": 16.2, "output_time_split": { "reasoning_pct": 53.5, "content_pct": 46.2, "tool_call_pct": 0.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "grok/grok-4.6": { "accuracy": 11.111, "latency": null, "stderr": 11.111, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 48.6, "avg_query_seconds": 33.7, "median_query_seconds": 24.7, "avg_ttft_seconds": 5.4, "output_time_split": { "reasoning_pct": 78.8, "content_pct": 21.2, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "meta/muse-wonder-full": { "accuracy": 11.111, "latency": null, "stderr": 11.111, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 185.3, "avg_query_seconds": 46.0, "median_query_seconds": 21.8, "avg_ttft_seconds": 43.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-sol": { "accuracy": 11.111, "latency": null, "stderr": 11.111, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 79.1, "avg_query_seconds": 18.8, "median_query_seconds": 13.4, "avg_ttft_seconds": 9.2, "output_time_split": { "reasoning_pct": 16.3, "content_pct": 23.7, "tool_call_pct": 60.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-6-luna": { "accuracy": 11.111, "latency": null, "stderr": 11.111, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 56.6, "avg_query_seconds": 28.9, "median_query_seconds": 17.2, "avg_ttft_seconds": 7.4, "output_time_split": { "reasoning_pct": 39.7, "content_pct": 3.1, "tool_call_pct": 57.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 321.6, "avg_query_seconds": 42.0, "median_query_seconds": 18.0, "avg_ttft_seconds": 24.8, "output_time_split": { "reasoning_pct": 75.2, "content_pct": 1.7, "tool_call_pct": 23.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 204.4, "avg_query_seconds": 56.1, "median_query_seconds": 36.5, "avg_ttft_seconds": 11.6, "output_time_split": { "reasoning_pct": 78.3, "content_pct": 0.2, "tool_call_pct": 21.5 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "Mini-SWE-agent" }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 172.4, "avg_query_seconds": 31.6, "median_query_seconds": 13.9, "avg_ttft_seconds": 27.8, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 154.7, "avg_query_seconds": 12.5, "median_query_seconds": 6.9, "avg_ttft_seconds": 10.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 91.9, "avg_query_seconds": 42.0, "median_query_seconds": 30.9, "avg_ttft_seconds": 32.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 287.4, "avg_query_seconds": 17.6, "median_query_seconds": 8.3, "avg_ttft_seconds": 16.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 186.9, "avg_query_seconds": 9.1, "median_query_seconds": 6.7, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 78.6, "content_pct": 0.5, "tool_call_pct": 20.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 282.6, "avg_query_seconds": 12.0, "median_query_seconds": 8.7, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 65.4, "content_pct": 0.4, "tool_call_pct": 34.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 169.6, "avg_query_seconds": 10.0, "median_query_seconds": 7.9, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 78.4, "content_pct": 0.2, "tool_call_pct": 21.4 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "Mini-SWE-agent" }, "fireworks/ember-1": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 334.8, "avg_query_seconds": 24.4, "median_query_seconds": 14.3, "avg_ttft_seconds": 6.2, "output_time_split": { "reasoning_pct": 66.9, "content_pct": 1.4, "tool_call_pct": 31.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 73.8, "avg_query_seconds": 13.5, "median_query_seconds": 8.2, "avg_ttft_seconds": 9.5, "output_time_split": { "reasoning_pct": 70.5, "content_pct": 29.1, "tool_call_pct": 0.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 61.1, "avg_query_seconds": 10.9, "median_query_seconds": 6.2, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 55.8, "content_pct": 43.9, "tool_call_pct": 0.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 70.4, "avg_query_seconds": 7.7, "median_query_seconds": 6.1, "avg_ttft_seconds": 2.9, "output_time_split": { "reasoning_pct": 99.3, "content_pct": 0.0, "tool_call_pct": 0.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 103.8, "avg_query_seconds": 6.8, "median_query_seconds": 5.3, "avg_ttft_seconds": 4.2, "output_time_split": { "reasoning_pct": 86.8, "content_pct": 11.7, "tool_call_pct": 1.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "Mini-SWE-agent" }, "grok/grok-4.7": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 82.1, "avg_query_seconds": 29.6, "median_query_seconds": 23.7, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 92.8, "content_pct": 7.2, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "Mini-SWE-agent" }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 50.0, "avg_query_seconds": 2.6, "median_query_seconds": 2.1, "avg_ttft_seconds": 1.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "Mini-SWE-agent" }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 244.8, "avg_query_seconds": 66.1, "median_query_seconds": 50.5, "avg_ttft_seconds": 25.0, "output_time_split": { "reasoning_pct": 70.2, "content_pct": 2.4, "tool_call_pct": 27.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "Mini-SWE-agent" }, "meta/muse-wonder": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 227.0, "avg_query_seconds": 16.2, "median_query_seconds": 9.2, "avg_ttft_seconds": 14.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 128.0, "avg_query_seconds": 10.2, "median_query_seconds": 7.2, "avg_ttft_seconds": 5.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 128.3, "avg_query_seconds": 16.0, "median_query_seconds": 11.0, "avg_ttft_seconds": 6.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 125.0, "avg_query_seconds": 10.6, "median_query_seconds": 7.8, "avg_ttft_seconds": 5.5, "output_time_split": { "reasoning_pct": 26.1, "content_pct": 1.4, "tool_call_pct": 72.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 98.3, "avg_query_seconds": 19.8, "median_query_seconds": 14.0, "avg_ttft_seconds": 8.4, "output_time_split": { "reasoning_pct": 28.6, "content_pct": 7.2, "tool_call_pct": 64.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Mini-SWE-agent" }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": "Mini-SWE-agent" }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 174.9, "avg_query_seconds": 26.5, "median_query_seconds": 14.3, "avg_ttft_seconds": 4.9, "output_time_split": { "reasoning_pct": 79.9, "content_pct": 0.6, "tool_call_pct": 19.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "Mini-SWE-agent" }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 157.8, "avg_query_seconds": 35.6, "median_query_seconds": 12.7, "avg_ttft_seconds": 5.4, "output_time_split": { "reasoning_pct": 62.3, "content_pct": 37.7, "tool_call_pct": 0.0 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "Terminus 2" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 126.7, "avg_query_seconds": 53.7, "median_query_seconds": 30.8, "avg_ttft_seconds": 13.4, "output_time_split": { "reasoning_pct": 84.8, "content_pct": 0.8, "tool_call_pct": 14.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 133.6, "avg_query_seconds": 59.7, "median_query_seconds": 33.5, "avg_ttft_seconds": 11.5, "output_time_split": { "reasoning_pct": 84.3, "content_pct": 1.4, "tool_call_pct": 14.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "Mini-SWE-agent" }, "zai/glm-5.3": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 246.7, "avg_query_seconds": 27.4, "median_query_seconds": 13.8, "avg_ttft_seconds": 6.9, "output_time_split": { "reasoning_pct": 99.1, "content_pct": 0.9, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "Mini-SWE-agent" }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "turn_metrics": { "tasks_measured": 9, "avg_turns": 166.0, "avg_query_seconds": 43.5, "median_query_seconds": 26.4, "avg_ttft_seconds": 6.8, "output_time_split": { "reasoning_pct": 98.8, "content_pct": 1.2, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "Mini-SWE-agent" } } } }