{ "metadata": { "benchmark": "ClaimBench", "slug": "claimbench", "description": "Does a 10-K excerpt support a claim, contradict it, or say nothing about it?", "benchmark_id": "claimbench", "family": "claimbench", "version": "1", "updated": "2026-10-20", "dataset_type": "public", "industry": "finance", "tasks": { "overall": "Overall", "direct_support": "Direct support", "missing_evidence": "Missing evidence", "non_supporting_citation": "Non-supporting citation", "numeric_discrepancy": "Numeric discrepancy", "subtle_contradiction": "Subtle contradiction" }, "models": [ "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "decision/classify-1", "decision/clef-omni", "decision/d1", "decision/decider-flash-v1.2", "decision/decider-v1.1-27b", "decision/decider-v1.2", "decision/drex-v1.5", "decision/drex-v2.0", "decision/gpt-6-luna", "decision/jev-1.13.0", "decision/jev-1.14", "decision/mercury-decide", "decision/microsoft-decision-1", "decision/tev1-4b-experimental", "openai/gpt-6-luna", "openai/gpt-6.1-sol" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "cost_label": "Cost / Claim", "cost_decimals": 6, "pareto_subset": { "label": "Decision models", "key_prefix": "decision/" }, "pareto_focus": true, "runner": "external", "mode": "one-shot", "archived": false, "partner": false, "total_models": 19 }, "tasks": { "overall": { "anthropic/claude-sonnet-5-5": { "accuracy": 100.0, "latency": 4.047, "stderr": 0.0, "cost_per_test": 0.045922, "token_totals": { "input_tokens": 33103510, "output_tokens": 725633, "reasoning_tokens": 595497, "cache_read_tokens": 0, "cache_write_tokens": 33087830 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 100.0, "latency": 5.059, "stderr": 0.0, "cost_per_test": 0.002541, "token_totals": { "input_tokens": 33103510, "output_tokens": 1684900, "reasoning_tokens": 1554532, "cache_read_tokens": 0, "cache_write_tokens": 33087830 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": 6.47, "stderr": 0.0, "cost_per_test": 0.091833, "token_totals": { "input_tokens": 33103510, "output_tokens": 724589, "reasoning_tokens": 594453, "cache_read_tokens": 0, "cache_write_tokens": 33087830 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 100.0, "latency": 11.262, "stderr": 0.0, "cost_per_test": 0.029239, "token_totals": { "input_tokens": 19723414, "output_tokens": 816345, "reasoning_tokens": 698171, "cache_read_tokens": 8006, "cache_write_tokens": 19425646 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/decider-v1.2": { "accuracy": 99.694, "latency": 1.229, "stderr": 0.125, "cost_per_test": 0.000104, "token_totals": { "input_tokens": 10184896, "output_tokens": 1960, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "openai/gpt-6-luna": { "accuracy": 99.184, "latency": 7.283, "stderr": 0.203, "cost_per_test": 0.001372, "token_totals": { "input_tokens": 19723414, "output_tokens": 462844, "reasoning_tokens": 349610, "cache_read_tokens": 0, "cache_write_tokens": 19433652 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/mercury-decide": { "accuracy": 99.082, "latency": 0.717, "stderr": 0.216, "cost_per_test": 0.000102, "token_totals": { "input_tokens": 9983693, "output_tokens": 1960, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inception via OpenRouter", "harness": null }, "decision/decider-v1.1-27b": { "accuracy": 98.776, "latency": 0.51, "stderr": 0.248, "cost_per_test": 0.000104, "token_totals": { "input_tokens": 10190776, "output_tokens": 1960, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/d1": { "accuracy": 98.673, "latency": 0.383, "stderr": 0.258, "cost_per_test": 0.000101, "token_totals": { "input_tokens": 9873790, "output_tokens": 0, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Liquid AI via OpenRouter", "harness": null }, "decision/microsoft-decision-1": { "accuracy": 98.214, "latency": 0.325, "stderr": 0.299, "cost_per_test": 0.000218, "token_totals": { "input_tokens": 10150283, "output_tokens": 1960, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Microsoft via OpenRouter", "harness": null }, "decision/jev-1.14": { "accuracy": 98.061, "latency": 0.184, "stderr": 0.312, "cost_per_test": 0.000233, "token_totals": { "input_tokens": 10856490, "output_tokens": 101037, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/drex-v1.5": { "accuracy": 98.01, "latency": 0.758, "stderr": 0.316, "cost_per_test": 0.000205, "token_totals": { "input_tokens": 10063376, "output_tokens": 138234, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/jev-1.13.0": { "accuracy": 97.653, "latency": 0.207, "stderr": 0.342, "cost_per_test": 0.000233, "token_totals": { "input_tokens": 10856490, "output_tokens": 101004, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/drex-v2.0": { "accuracy": 97.296, "latency": 0.587, "stderr": 0.366, "cost_per_test": 0.000308, "token_totals": { "input_tokens": 10057909, "output_tokens": 138217, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/gpt-6-luna": { "accuracy": 96.888, "latency": 0.2, "stderr": 0.392, "cost_per_test": 0.000501, "token_totals": { "input_tokens": 9816552, "output_tokens": 0, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/decider-flash-v1.2": { "accuracy": 96.531, "latency": 1.088, "stderr": 0.413, "cost_per_test": 5.2e-05, "token_totals": { "input_tokens": 10184896, "output_tokens": 1960, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/tev1-4b-experimental": { "accuracy": 89.796, "latency": 0.419, "stderr": 0.684, "cost_per_test": 0.00022, "token_totals": { "input_tokens": 10279643, "output_tokens": 3920, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI via OpenRouter", "harness": null }, "decision/clef-omni": { "accuracy": 87.704, "latency": 0.421, "stderr": 0.742, "cost_per_test": 0.000783, "token_totals": { "input_tokens": 10228771, "output_tokens": 0, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cloudflare via OpenRouter", "harness": null }, "decision/classify-1": { "accuracy": 85.0, "latency": 1.243, "stderr": 0.807, "cost_per_test": 0.000215, "token_totals": { "input_tokens": 10537696, "output_tokens": 1960, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "ScaleDown via OpenRouter", "harness": null } }, "direct_support": { "decision/decider-v1.1-27b": { "accuracy": 100.0, "latency": 0.492809, "stderr": 0.0, "cost_per_test": 0.0001, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/decider-flash-v1.2": { "accuracy": 100.0, "latency": 1.098895, "stderr": 0.0, "cost_per_test": 5e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/decider-v1.2": { "accuracy": 100.0, "latency": 1.142273, "stderr": 0.0, "cost_per_test": 0.0001, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 100.0, "latency": 3.916934, "stderr": 0.0, "cost_per_test": 0.044561, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 100.0, "latency": 4.887758, "stderr": 0.0, "cost_per_test": 0.002461, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": 6.470569, "stderr": 0.0, "cost_per_test": 0.089478, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 100.0, "latency": 6.911378, "stderr": 0.0, "cost_per_test": 0.00132, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 100.0, "latency": 10.656061, "stderr": 0.0, "cost_per_test": 0.028083, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/microsoft-decision-1": { "accuracy": 99.745, "latency": 0.314485, "stderr": 0.255, "cost_per_test": 0.000209, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Microsoft via OpenRouter", "harness": null }, "decision/jev-1.14": { "accuracy": 99.49, "latency": 0.179946, "stderr": 0.36, "cost_per_test": 0.000225, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/jev-1.13.0": { "accuracy": 99.49, "latency": 0.19949, "stderr": 0.36, "cost_per_test": 0.000225, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/drex-v1.5": { "accuracy": 99.235, "latency": 0.754617, "stderr": 0.441, "cost_per_test": 0.000198, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/d1": { "accuracy": 98.98, "latency": 0.391921, "stderr": 0.508, "cost_per_test": 9.7e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Liquid AI via OpenRouter", "harness": null }, "decision/drex-v2.0": { "accuracy": 98.724, "latency": 0.562635, "stderr": 0.568, "cost_per_test": 0.000295, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/tev1-4b-experimental": { "accuracy": 98.469, "latency": 0.414926, "stderr": 0.621, "cost_per_test": 0.000212, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI via OpenRouter", "harness": null }, "decision/gpt-6-luna": { "accuracy": 98.214, "latency": 0.190821, "stderr": 0.67, "cost_per_test": 0.000482, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/clef-omni": { "accuracy": 98.214, "latency": 0.382556, "stderr": 0.67, "cost_per_test": 0.000754, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cloudflare via OpenRouter", "harness": null }, "decision/mercury-decide": { "accuracy": 97.959, "latency": 0.681495, "stderr": 0.715, "cost_per_test": 9.8e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inception via OpenRouter", "harness": null }, "decision/classify-1": { "accuracy": 88.01, "latency": 1.218467, "stderr": 1.643, "cost_per_test": 0.000207, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "ScaleDown via OpenRouter", "harness": null } }, "missing_evidence": { "decision/jev-1.14": { "accuracy": 100.0, "latency": 0.187191, "stderr": 0.0, "cost_per_test": 0.000237, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/gpt-6-luna": { "accuracy": 100.0, "latency": 0.207684, "stderr": 0.0, "cost_per_test": 0.000511, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/jev-1.13.0": { "accuracy": 100.0, "latency": 0.216273, "stderr": 0.0, "cost_per_test": 0.000237, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/microsoft-decision-1": { "accuracy": 100.0, "latency": 0.338566, "stderr": 0.0, "cost_per_test": 0.000221, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Microsoft via OpenRouter", "harness": null }, "decision/d1": { "accuracy": 100.0, "latency": 0.364704, "stderr": 0.0, "cost_per_test": 0.000103, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Liquid AI via OpenRouter", "harness": null }, "decision/decider-v1.1-27b": { "accuracy": 100.0, "latency": 0.525054, "stderr": 0.0, "cost_per_test": 0.000106, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/mercury-decide": { "accuracy": 100.0, "latency": 0.724819, "stderr": 0.0, "cost_per_test": 0.000104, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inception via OpenRouter", "harness": null }, "decision/drex-v1.5": { "accuracy": 100.0, "latency": 0.801066, "stderr": 0.0, "cost_per_test": 0.000209, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/decider-flash-v1.2": { "accuracy": 100.0, "latency": 1.15412, "stderr": 0.0, "cost_per_test": 5.3e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/decider-v1.2": { "accuracy": 100.0, "latency": 1.172673, "stderr": 0.0, "cost_per_test": 0.000106, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 100.0, "latency": 4.071778, "stderr": 0.0, "cost_per_test": 0.046719, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 100.0, "latency": 5.214852, "stderr": 0.0, "cost_per_test": 0.002589, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": 6.906543, "stderr": 0.0, "cost_per_test": 0.09404, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 100.0, "latency": 7.086452, "stderr": 0.0, "cost_per_test": 0.001385, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 100.0, "latency": 12.166446, "stderr": 0.0, "cost_per_test": 0.030066, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/tev1-4b-experimental": { "accuracy": 99.745, "latency": 0.414602, "stderr": 0.255, "cost_per_test": 0.000224, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI via OpenRouter", "harness": null }, "decision/clef-omni": { "accuracy": 99.49, "latency": 0.419969, "stderr": 0.36, "cost_per_test": 0.000797, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cloudflare via OpenRouter", "harness": null }, "decision/drex-v2.0": { "accuracy": 98.214, "latency": 0.621497, "stderr": 0.67, "cost_per_test": 0.000314, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/classify-1": { "accuracy": 97.704, "latency": 1.237679, "stderr": 0.757, "cost_per_test": 0.000219, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "ScaleDown via OpenRouter", "harness": null } }, "non_supporting_citation": { "decision/gpt-6-luna": { "accuracy": 100.0, "latency": 0.193136, "stderr": 0.0, "cost_per_test": 0.000508, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/microsoft-decision-1": { "accuracy": 100.0, "latency": 0.329074, "stderr": 0.0, "cost_per_test": 0.00022, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Microsoft via OpenRouter", "harness": null }, "decision/d1": { "accuracy": 100.0, "latency": 0.343757, "stderr": 0.0, "cost_per_test": 0.000102, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Liquid AI via OpenRouter", "harness": null }, "decision/decider-v1.1-27b": { "accuracy": 100.0, "latency": 0.515857, "stderr": 0.0, "cost_per_test": 0.000105, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/decider-v1.2": { "accuracy": 100.0, "latency": 1.277386, "stderr": 0.0, "cost_per_test": 0.000105, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 100.0, "latency": 4.133826, "stderr": 0.0, "cost_per_test": 0.04655, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 100.0, "latency": 5.270486, "stderr": 0.0, "cost_per_test": 0.002589, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": 6.912532, "stderr": 0.0, "cost_per_test": 0.094023, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 100.0, "latency": 7.176003, "stderr": 0.0, "cost_per_test": 0.001385, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 100.0, "latency": 12.071575, "stderr": 0.0, "cost_per_test": 0.029973, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/mercury-decide": { "accuracy": 99.744, "latency": 0.745668, "stderr": 0.256, "cost_per_test": 0.000103, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inception via OpenRouter", "harness": null }, "decision/decider-flash-v1.2": { "accuracy": 99.744, "latency": 1.159611, "stderr": 0.256, "cost_per_test": 5.3e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/jev-1.14": { "accuracy": 99.488, "latency": 0.169038, "stderr": 0.361, "cost_per_test": 0.000235, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/drex-v1.5": { "accuracy": 99.233, "latency": 0.750647, "stderr": 0.442, "cost_per_test": 0.000208, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/jev-1.13.0": { "accuracy": 98.721, "latency": 0.203184, "stderr": 0.569, "cost_per_test": 0.000235, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/tev1-4b-experimental": { "accuracy": 98.21, "latency": 0.427593, "stderr": 0.671, "cost_per_test": 0.000223, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI via OpenRouter", "harness": null }, "decision/classify-1": { "accuracy": 97.698, "latency": 1.092724, "stderr": 0.759, "cost_per_test": 0.000218, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "ScaleDown via OpenRouter", "harness": null }, "decision/drex-v2.0": { "accuracy": 97.442, "latency": 0.584325, "stderr": 0.799, "cost_per_test": 0.000312, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/clef-omni": { "accuracy": 96.931, "latency": 0.434, "stderr": 0.873, "cost_per_test": 0.000793, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cloudflare via OpenRouter", "harness": null } }, "numeric_discrepancy": { "decision/decider-v1.1-27b": { "accuracy": 100.0, "latency": 0.535969, "stderr": 0.0, "cost_per_test": 0.000108, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/mercury-decide": { "accuracy": 100.0, "latency": 0.751333, "stderr": 0.0, "cost_per_test": 0.000106, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inception via OpenRouter", "harness": null }, "decision/decider-v1.2": { "accuracy": 100.0, "latency": 1.267786, "stderr": 0.0, "cost_per_test": 0.000108, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 100.0, "latency": 4.065911, "stderr": 0.0, "cost_per_test": 0.047162, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 100.0, "latency": 4.605756, "stderr": 0.0, "cost_per_test": 0.002557, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": 5.872794, "stderr": 0.0, "cost_per_test": 0.093144, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 100.0, "latency": 7.372316, "stderr": 0.0, "cost_per_test": 0.001414, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 100.0, "latency": 10.514776, "stderr": 0.0, "cost_per_test": 0.029731, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/d1": { "accuracy": 99.746, "latency": 0.398641, "stderr": 0.254, "cost_per_test": 0.000104, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Liquid AI via OpenRouter", "harness": null }, "decision/microsoft-decision-1": { "accuracy": 99.491, "latency": 0.326982, "stderr": 0.359, "cost_per_test": 0.000226, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Microsoft via OpenRouter", "harness": null }, "decision/drex-v1.5": { "accuracy": 98.982, "latency": 0.721677, "stderr": 0.507, "cost_per_test": 0.000214, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/decider-flash-v1.2": { "accuracy": 98.982, "latency": 0.987995, "stderr": 0.507, "cost_per_test": 5.4e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/jev-1.14": { "accuracy": 98.728, "latency": 0.193224, "stderr": 0.566, "cost_per_test": 0.000241, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/gpt-6-luna": { "accuracy": 98.219, "latency": 0.199888, "stderr": 0.668, "cost_per_test": 0.000518, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/jev-1.13.0": { "accuracy": 97.964, "latency": 0.211771, "stderr": 0.713, "cost_per_test": 0.000241, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/drex-v2.0": { "accuracy": 97.964, "latency": 0.584061, "stderr": 0.713, "cost_per_test": 0.00032, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/clef-omni": { "accuracy": 88.041, "latency": 0.455873, "stderr": 1.639, "cost_per_test": 0.000814, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cloudflare via OpenRouter", "harness": null }, "decision/tev1-4b-experimental": { "accuracy": 83.969, "latency": 0.426303, "stderr": 1.853, "cost_per_test": 0.000229, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI via OpenRouter", "harness": null }, "decision/classify-1": { "accuracy": 74.809, "latency": 1.429132, "stderr": 2.193, "cost_per_test": 0.000223, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "ScaleDown via OpenRouter", "harness": null } }, "subtle_contradiction": { "anthropic/claude-sonnet-5-5": { "accuracy": 100.0, "latency": 4.048482, "stderr": 0.0, "cost_per_test": 0.044616, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 100.0, "latency": 5.318388, "stderr": 0.0, "cost_per_test": 0.002508, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": 6.187811, "stderr": 0.0, "cost_per_test": 0.088483, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 100.0, "latency": 10.905898, "stderr": 0.0, "cost_per_test": 0.028341, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/decider-v1.2": { "accuracy": 98.469, "latency": 1.283758, "stderr": 0.621, "cost_per_test": 0.0001, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/mercury-decide": { "accuracy": 97.704, "latency": 0.682528, "stderr": 0.757, "cost_per_test": 9.8e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inception via OpenRouter", "harness": null }, "openai/gpt-6-luna": { "accuracy": 95.918, "latency": 7.866128, "stderr": 1.001, "cost_per_test": 0.001357, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/d1": { "accuracy": 94.643, "latency": 0.417574, "stderr": 1.139, "cost_per_test": 9.8e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Liquid AI via OpenRouter", "harness": null }, "decision/drex-v2.0": { "accuracy": 94.133, "latency": 0.582569, "stderr": 1.189, "cost_per_test": 0.000298, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/decider-v1.1-27b": { "accuracy": 93.878, "latency": 0.477852, "stderr": 1.212, "cost_per_test": 0.000101, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/jev-1.14": { "accuracy": 92.602, "latency": 0.189967, "stderr": 1.324, "cost_per_test": 0.000225, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/drex-v1.5": { "accuracy": 92.602, "latency": 0.759908, "stderr": 1.324, "cost_per_test": 0.000198, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nace.AI via OpenRouter", "harness": null }, "decision/jev-1.13.0": { "accuracy": 92.092, "latency": 0.20212, "stderr": 1.365, "cost_per_test": 0.000225, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "TypeSafe via OpenRouter", "harness": null }, "decision/microsoft-decision-1": { "accuracy": 91.837, "latency": 0.313668, "stderr": 1.385, "cost_per_test": 0.00021, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Microsoft via OpenRouter", "harness": null }, "decision/gpt-6-luna": { "accuracy": 88.01, "latency": 0.20688, "stderr": 1.643, "cost_per_test": 0.000485, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "decision/decider-flash-v1.2": { "accuracy": 83.929, "latency": 1.039219, "stderr": 1.857, "cost_per_test": 5e-05, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Perplexity via OpenRouter", "harness": null }, "decision/tev1-4b-experimental": { "accuracy": 68.622, "latency": 0.409148, "stderr": 2.347, "cost_per_test": 0.000213, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI via OpenRouter", "harness": null }, "decision/classify-1": { "accuracy": 66.837, "latency": 1.237536, "stderr": 2.381, "cost_per_test": 0.000208, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "ScaleDown via OpenRouter", "harness": null }, "decision/clef-omni": { "accuracy": 55.867, "latency": 0.41201, "stderr": 2.511, "cost_per_test": 0.000757, "temperature": null, "top_p": null, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cloudflare via OpenRouter", "harness": null } } } }