{ "metadata": { "benchmark": "Vals Router Index", "slug": "vals_router_index", "description": "How model routers compare with single models on long agentic work from the Vals Index.", "benchmark_id": "vals_router_index", "family": "vals_router_index", "version": "1.0", "updated": "2026-10-20", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent", "emb": "EMB", "terminal_bench_4": "Terminal-Bench 4.0", "vibe_code_bench": "Vibe Code Bench", "code_migration": "Code Migration", "legal_research": "Legal Research", "legal_agent_benchmark": "HLAB", "tax_agent_bench": "Tax Agent Bench" }, "models": [ "fireworks/firerouter", "openrouter/openrouter/auto", "openrouter/typesafe/jev-router", "ramp/switchyard", "sakana/fugu-max-v1.0" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 5 }, "tasks": { "overall": { "openrouter/openrouter/auto": { "accuracy": 52.195, "latency": 2420.133, "stderr": 1.207, "cost_per_test": 0.817334, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 56.458, "latency": 2807.558, "stderr": 1.303, "cost_per_test": 9.217354, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 57.795, "latency": 680.404, "stderr": 1.167, "cost_per_test": 0.956238, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 55.652, "latency": 1478.241, "stderr": 1.332, "cost_per_test": 5.303181, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 48.114, "latency": 850.32, "stderr": 1.166, "cost_per_test": 3.912391, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "finance_agent": { "openrouter/openrouter/auto": { "accuracy": 59.098, "latency": 268.433, "stderr": 2.201, "cost_per_test": 0.09512, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 64.456, "latency": 147.97, "stderr": 2.14, "cost_per_test": 0.653802, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 62.451, "latency": 127.076, "stderr": 2.145, "cost_per_test": 0.145253, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 59.614, "latency": 261.498, "stderr": 2.183, "cost_per_test": 0.895878, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 54.99, "latency": 177.443, "stderr": 2.217, "cost_per_test": 0.952185, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "emb": { "openrouter/openrouter/auto": { "accuracy": 67.174, "latency": 902.021, "stderr": 1.95, "cost_per_test": 0.204939, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 73.016, "latency": 731.624, "stderr": 1.895, "cost_per_test": 2.002952, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 69.61, "latency": 354.757, "stderr": 2.223, "cost_per_test": 0.378797, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 70.977, "latency": 551.08, "stderr": 2.23, "cost_per_test": 2.029949, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 66.927, "latency": 460.153, "stderr": 2.213, "cost_per_test": 0.854517, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "terminal_bench_4": { "openrouter/openrouter/auto": { "accuracy": 25.758, "latency": 2604.973, "stderr": 4.058, "cost_per_test": 0.862076, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 46.465, "latency": 3465.113, "stderr": 5.38, "cost_per_test": 7.274694, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 32.323, "latency": 1184.575, "stderr": 4.772, "cost_per_test": 1.089203, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 45.96, "latency": 1353.451, "stderr": 5.178, "cost_per_test": 2.375726, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 17.172, "latency": 1901.851, "stderr": 3.616, "cost_per_test": 8.364969, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "vibe_code_bench": { "openrouter/openrouter/auto": { "accuracy": 72.257, "latency": 2642.737, "stderr": 4.666, "cost_per_test": 1.002777, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 78.786, "latency": 1205.389, "stderr": 4.649, "cost_per_test": 6.51642, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 86.475, "latency": 1309.887, "stderr": 2.124, "cost_per_test": 3.16861, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 61.391, "latency": 856.124, "stderr": 5.535, "cost_per_test": 4.551245, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 69.608, "latency": 1750.313, "stderr": 4.14, "cost_per_test": 15.437519, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "code_migration": { "openrouter/openrouter/auto": { "accuracy": 44.815, "latency": 10049.686, "stderr": 4.266, "cost_per_test": 3.736539, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 21.25, "latency": 14502.675, "stderr": 4.871, "cost_per_test": 47.473857, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 52.61, "latency": 1451.158, "stderr": 4.261, "cost_per_test": 1.215162, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 54.683, "latency": 6605.332, "stderr": 3.918, "cost_per_test": 26.506681, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 33.946, "latency": 1391.26, "stderr": 3.816, "cost_per_test": 2.702462, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "legal_research": { "openrouter/openrouter/auto": { "accuracy": 29.808, "latency": 1808.769, "stderr": 3.179, "cost_per_test": 0.207931, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 41.346, "latency": 383.829, "stderr": 3.423, "cost_per_test": 0.727174, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 36.538, "latency": 236.513, "stderr": 3.347, "cost_per_test": 0.258068, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 29.808, "latency": 415.838, "stderr": 3.179, "cost_per_test": 0.902983, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 24.519, "latency": 373.239, "stderr": 2.99, "cost_per_test": 0.713539, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "legal_agent_benchmark": { "openrouter/openrouter/auto": { "accuracy": 5.0, "latency": 713.15, "stderr": 1.814, "cost_per_test": 0.395779, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 6.25, "latency": 1850.28, "stderr": 2.012, "cost_per_test": 8.739914, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 5.0, "latency": 629.575, "stderr": 1.908, "cost_per_test": 1.251771, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 1.25, "latency": 1466.576, "stderr": 0.929, "cost_per_test": 4.470621, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 2.917, "latency": 486.197, "stderr": 1.226, "cost_per_test": 1.779858, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "tax_agent_bench": { "openrouter/openrouter/auto": { "accuracy": 11.996, "latency": 371.294, "stderr": 2.308, "cost_per_test": 0.03351, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "fireworks/firerouter": { "accuracy": 23.796, "latency": 173.584, "stderr": 3.07, "cost_per_test": 0.350018, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ramp/switchyard": { "accuracy": 22.28, "latency": 149.691, "stderr": 3.003, "cost_per_test": 0.143038, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openrouter/typesafe/jev-router": { "accuracy": 16.036, "latency": 316.031, "stderr": 2.646, "cost_per_test": 0.692363, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "sakana/fugu-max-v1.0": { "accuracy": 13.316, "latency": 262.102, "stderr": 2.44, "cost_per_test": 0.494077, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } } } }