{
  "metadata": {
    "benchmark": "Vals Web Search Index",
    "slug": "web_search_backends",
    "description": "Comparing native provider search against independent web-search tools on legal-research and finance-analysis tasks",
    "benchmark_id": "web_search_backends",
    "family": "web_search",
    "version": "1",
    "updated": "2026-10-21",
    "dataset_type": "private",
    "industry": "index",
    "tasks": {
      "overall": "Overall",
      "finance": "Finance Analysis",
      "legal": "Legal Research"
    },
    "models": [
      "anthropic/claude-fable-5",
      "anthropic/claude-fable-5-exa",
      "google/gemini-3.5-flash",
      "google/gemini-3.5-flash-exa",
      "grok/grok-4.5",
      "grok/grok-4.5-exa",
      "openai/gpt-5.6-sol",
      "openai/gpt-5.6-sol-exa",
      "openai/gpt-5.6-sol-keenable"
    ],
    "partners": [],
    "showBadge": false,
    "visible": true,
    "use_cost_per_test": true,
    "cost_label": "Cost/Task",
    "harness_label": "Search Tool",
    "runner": "external",
    "mode": "agentic",
    "archived": false,
    "partner": false,
    "total_models": 9
  },
  "tasks": {
    "overall": {
      "openai/gpt-5.6-sol-keenable": {
        "accuracy": 49.076695,
        "latency": 395.278729,
        "stderr": 2.020759,
        "cost_per_test": 8.842075,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Keenable",
        "harness_config": "Keenable Pro search"
      },
      "anthropic/claude-fable-5-exa": {
        "accuracy": 48.454,
        "latency": 351.537,
        "stderr": 2.031857832132947,
        "cost_per_test": 3.481848,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": "max",
        "provider": "Anthropic",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "anthropic/claude-fable-5": {
        "accuracy": 46.93805588941155,
        "latency": 742.4488012222541,
        "stderr": 2.099818988794454,
        "cost_per_test": 4.368756,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": "max",
        "provider": "Anthropic",
        "harness": "Native",
        "harness_config": "Anthropic web search tool"
      },
      "openai/gpt-5.6-sol-exa": {
        "accuracy": 45.2405,
        "latency": 530.7515,
        "stderr": 2.0060384467900905,
        "cost_per_test": 18.968851,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "openai/gpt-5.6-sol": {
        "accuracy": 43.58411538461539,
        "latency": 328.83156009615385,
        "stderr": 1.9821316327108887,
        "cost_per_test": 0.953765,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Native",
        "harness_config": "OpenAI web search tool"
      },
      "google/gemini-3.5-flash-exa": {
        "accuracy": 41.361000000000004,
        "latency": 197.4605,
        "stderr": 1.930917139599729,
        "cost_per_test": 1.177103,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 65536,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "Google",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "grok/grok-4.5-exa": {
        "accuracy": 38.747,
        "latency": 100.9255,
        "stderr": 1.945497365713971,
        "cost_per_test": 0.576384,
        "temperature": 0.7,
        "top_p": 0.95,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "xAI",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "google/gemini-3.5-flash": {
        "accuracy": 37.040653846153845,
        "latency": 249.53663461538463,
        "stderr": 1.8547389298089656,
        "cost_per_test": 0.499017,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 65536,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "Google",
        "harness": "Native",
        "harness_config": "Google Search grounding"
      },
      "grok/grok-4.5": {
        "accuracy": 35.5375,
        "latency": 137.042,
        "stderr": 1.9338961321642898,
        "cost_per_test": 0.991213,
        "temperature": 0.7,
        "top_p": 0.95,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "xAI",
        "harness": "Native",
        "harness_config": "xAI web search tool"
      }
    },
    "finance": {
      "openai/gpt-5.6-sol-keenable": {
        "accuracy": 53.922621,
        "latency": 316.600789,
        "stderr": 2.101753,
        "cost_per_test": 5.483549,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Keenable",
        "harness_config": "Keenable Pro search"
      },
      "anthropic/claude-fable-5-exa": {
        "accuracy": 53.639,
        "latency": 270.746,
        "stderr": 2.157,
        "cost_per_test": 2.921163,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": "max",
        "provider": "Anthropic",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "openai/gpt-5.6-sol-exa": {
        "accuracy": 52.019,
        "latency": 396.844,
        "stderr": 2.16,
        "cost_per_test": 9.661596,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "google/gemini-3.5-flash-exa": {
        "accuracy": 51.953,
        "latency": 251.846,
        "stderr": 2.15,
        "cost_per_test": 1.637114,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 65536,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "Google",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "openai/gpt-5.6-sol": {
        "accuracy": 50.149,
        "latency": 376.817,
        "stderr": 2.11,
        "cost_per_test": 0.951829,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Native",
        "harness_config": "OpenAI web search tool"
      },
      "google/gemini-3.5-flash": {
        "accuracy": 47.639,
        "latency": 386.593,
        "stderr": 2.089,
        "cost_per_test": 0.73047,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 65536,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "Google",
        "harness": "Native",
        "harness_config": "Google Search grounding"
      },
      "anthropic/claude-fable-5": {
        "accuracy": 44.83765024036157,
        "latency": 805.821472636816,
        "stderr": 2.3588410103167337,
        "cost_per_test": 5.045034,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": "max",
        "provider": "Anthropic",
        "harness": "Native",
        "harness_config": "Anthropic web search tool"
      },
      "grok/grok-4.5-exa": {
        "accuracy": 41.917,
        "latency": 122.317,
        "stderr": 2.016,
        "cost_per_test": 0.518534,
        "temperature": 0.7,
        "top_p": 0.95,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "xAI",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "grok/grok-4.5": {
        "accuracy": 32.133,
        "latency": 206.246,
        "stderr": 1.864,
        "cost_per_test": 1.525011,
        "temperature": 0.7,
        "top_p": 0.95,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "xAI",
        "harness": "Native",
        "harness_config": "xAI web search tool"
      }
    },
    "legal": {
      "anthropic/claude-fable-5": {
        "accuracy": 49.03846153846153,
        "latency": 679.0761298076923,
        "stderr": 3.4745975638705247,
        "cost_per_test": 3.692479,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": "max",
        "provider": "Anthropic",
        "harness": "Native",
        "harness_config": "Anthropic web search tool"
      },
      "openai/gpt-5.6-sol-keenable": {
        "accuracy": 44.230769,
        "latency": 473.956668,
        "stderr": 3.452029,
        "cost_per_test": 12.2006,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Keenable",
        "harness_config": "Keenable Pro search"
      },
      "anthropic/claude-fable-5-exa": {
        "accuracy": 43.269,
        "latency": 432.328,
        "stderr": 3.444,
        "cost_per_test": 4.042534,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": null,
        "verbosity": null,
        "compute_effort": "max",
        "provider": "Anthropic",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "grok/grok-4.5": {
        "accuracy": 38.942,
        "latency": 67.838,
        "stderr": 3.389,
        "cost_per_test": 0.457415,
        "temperature": 0.7,
        "top_p": 0.95,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "xAI",
        "harness": "Native",
        "harness_config": "xAI web search tool"
      },
      "openai/gpt-5.6-sol-exa": {
        "accuracy": 38.462,
        "latency": 664.659,
        "stderr": 3.381,
        "cost_per_test": 28.276106,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "openai/gpt-5.6-sol": {
        "accuracy": 37.019230769230774,
        "latency": 280.8461201923077,
        "stderr": 3.3560815302332174,
        "cost_per_test": 0.9557,
        "temperature": null,
        "top_p": null,
        "max_output_tokens": 128000,
        "reasoning": null,
        "reasoning_effort": "max",
        "verbosity": null,
        "compute_effort": null,
        "provider": "OpenAI",
        "harness": "Native",
        "harness_config": "OpenAI web search tool"
      },
      "grok/grok-4.5-exa": {
        "accuracy": 35.577,
        "latency": 79.534,
        "stderr": 3.328,
        "cost_per_test": 0.634233,
        "temperature": 0.7,
        "top_p": 0.95,
        "max_output_tokens": null,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "xAI",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "google/gemini-3.5-flash-exa": {
        "accuracy": 30.769,
        "latency": 143.075,
        "stderr": 3.208,
        "cost_per_test": 0.717092,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 65536,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "Google",
        "harness": "Exa",
        "harness_config": "Exa auto search with highlights"
      },
      "google/gemini-3.5-flash": {
        "accuracy": 26.442307692307693,
        "latency": 112.48026923076924,
        "stderr": 3.065339294596216,
        "cost_per_test": 0.267564,
        "temperature": 1.0,
        "top_p": null,
        "max_output_tokens": 65536,
        "reasoning": null,
        "reasoning_effort": "high",
        "verbosity": null,
        "compute_effort": null,
        "provider": "Google",
        "harness": "Native",
        "harness_config": "Google Search grounding"
      }
    }
  }
}