{
    "metadata": {
        "benchmark": "Access to Justice",
        "slug": "a2j",
        "description": "To what extent can AI agents guide an unrepresented client through a real legal problem?",
        "benchmark_id": "a2j",
        "family": "a2j",
        "version": "1",
        "updated": "2026-11-10",
        "dataset_type": "private",
        "industry": "social mobility",
        "tasks": {
            "overall": "Overall"
        },
        "models": [
            "alibaba/qwen3.8-max",
            "anthropic/claude-opus-5-5",
            "anthropic/claude-sonnet-5-5",
            "deepseek/deepseek-v4.1-flash",
            "deepseek/deepseek-v4.1-flash-1017",
            "google/gemini-3.8-flash",
            "google/gemini-4-argon",
            "grok/grok-4.6",
            "grok/grok-4.7",
            "kimi/kimi-k3",
            "kimi/kimi-k3.1-pro",
            "meta/muse-wonder",
            "meta/muse-wonder-full",
            "meta/muse_spark_1_3",
            "meta/muse_spark_1_3_max",
            "minimax/MiniMax-M3",
            "minimax/MiniMax-M3.1-flash",
            "openai/gpt-5.6-luna",
            "openai/gpt-6.1-sol",
            "xiaomi/mimo-v2.6-flash",
            "xiaomi/mimo-v2.6-pro",
            "zai/glm-5.3",
            "zai/glm-5.3-flash"
        ],
        "partners": [],
        "showBadge": false,
        "visible": true,
        "use_cost_per_test": true,
        "runner": "external",
        "mode": "agentic",
        "archived": false,
        "partner": false,
        "total_models": 23
    },
    "tasks": {
        "overall": {
            "alibaba/qwen3.8-max": {
                "accuracy": 76.368,
                "latency": 1096.445,
                "stderr": 0.684,
                "cost_per_test": 0.607832,
                "token_totals": {
                    "input_tokens": 220692464,
                    "output_tokens": 9078258,
                    "reasoning_tokens": 4891792,
                    "cache_read_tokens": 195817600,
                    "cache_write_tokens": null
                },
                "token_tasks_measured": 252,
                "temperature": 0.7,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Alibaba",
                "harness": null
            },
            "anthropic/claude-sonnet-5-5": {
                "accuracy": 74.942,
                "latency": 1053.897,
                "stderr": 0.698,
                "cost_per_test": 2.366309,
                "token_totals": {
                    "input_tokens": 744647379,
                    "output_tokens": 35251740,
                    "reasoning_tokens": 30733430,
                    "cache_read_tokens": 674091350,
                    "cache_write_tokens": 70542649
                },
                "token_tasks_measured": 252,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "meta/muse-wonder-full": {
                "accuracy": 74.676,
                "latency": 851.488,
                "stderr": 0.7,
                "cost_per_test": 1.240886,
                "token_totals": {
                    "input_tokens": 292620737,
                    "output_tokens": 6762517,
                    "reasoning_tokens": 2931050,
                    "cache_read_tokens": 261323720,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "anthropic/claude-opus-5-5": {
                "accuracy": 73.765,
                "latency": 950.408,
                "stderr": 0.709,
                "cost_per_test": 3.378892,
                "token_totals": {
                    "input_tokens": 624042854,
                    "output_tokens": 23478414,
                    "reasoning_tokens": 19684037,
                    "cache_read_tokens": 570476560,
                    "cache_write_tokens": 53551996
                },
                "token_tasks_measured": 252,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "Anthropic",
                "harness": null
            },
            "zai/glm-5.3": {
                "accuracy": 73.696,
                "latency": 690.135,
                "stderr": 0.709,
                "cost_per_test": 0.458045,
                "token_totals": {
                    "input_tokens": 160559246,
                    "output_tokens": 10357703,
                    "reasoning_tokens": 8145919,
                    "cache_read_tokens": 135903142,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Fireworks AI",
                "harness": null
            },
            "grok/grok-4.7": {
                "accuracy": 73.69,
                "latency": 467.295,
                "stderr": 0.709,
                "cost_per_test": 0.821038,
                "token_totals": {
                    "input_tokens": 237949808,
                    "output_tokens": 6231968,
                    "reasoning_tokens": 3957398,
                    "cache_read_tokens": 204259968,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "minimax/MiniMax-M3.1-flash": {
                "accuracy": 73.307,
                "latency": 594.035,
                "stderr": 0.712,
                "cost_per_test": 0.064516,
                "token_totals": {
                    "input_tokens": 306890703,
                    "output_tokens": 12491053,
                    "reasoning_tokens": 8493454,
                    "cache_read_tokens": 276075546,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": "max",
                "provider": "MiniMax",
                "harness": null
            },
            "deepseek/deepseek-v4.1-flash-1017": {
                "accuracy": 73.105,
                "latency": 269.066,
                "stderr": 0.714,
                "cost_per_test": 0.089166,
                "token_totals": {
                    "input_tokens": 300775133,
                    "output_tokens": 10719424,
                    "reasoning_tokens": 7766288,
                    "cache_read_tokens": 274238077,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 384000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "meta/muse-wonder": {
                "accuracy": 72.956,
                "latency": 285.753,
                "stderr": 0.716,
                "cost_per_test": 1.097287,
                "token_totals": {
                    "input_tokens": 273112674,
                    "output_tokens": 5753595,
                    "reasoning_tokens": 2414646,
                    "cache_read_tokens": 245001636,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "deepseek/deepseek-v4.1-flash": {
                "accuracy": 72.862,
                "latency": 559.654,
                "stderr": 0.716,
                "cost_per_test": 0.108645,
                "token_totals": {
                    "input_tokens": 325050965,
                    "output_tokens": 15680101,
                    "reasoning_tokens": 13053584,
                    "cache_read_tokens": 302560384,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 1.0,
                "max_output_tokens": 384000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "DeepSeek",
                "harness": null
            },
            "meta/muse_spark_1_3": {
                "accuracy": 72.179,
                "latency": 130.207,
                "stderr": 0.723,
                "cost_per_test": 0.183142,
                "token_totals": {
                    "input_tokens": 98379489,
                    "output_tokens": 4019181,
                    "reasoning_tokens": 1428663,
                    "cache_read_tokens": 85367396,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "xhigh",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "meta/muse_spark_1_3_max": {
                "accuracy": 72.152,
                "latency": 252.808,
                "stderr": 0.722,
                "cost_per_test": 0.215136,
                "token_totals": {
                    "input_tokens": 119967803,
                    "output_tokens": 4519751,
                    "reasoning_tokens": 1769956,
                    "cache_read_tokens": 104504021,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": null,
                "max_output_tokens": 131072,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Meta",
                "harness": null
            },
            "grok/grok-4.6": {
                "accuracy": 71.343,
                "latency": 373.466,
                "stderr": 0.729,
                "cost_per_test": 0.706889,
                "token_totals": {
                    "input_tokens": 211099368,
                    "output_tokens": 4016121,
                    "reasoning_tokens": 2016718,
                    "cache_read_tokens": 178772992,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 0.7,
                "top_p": 0.95,
                "max_output_tokens": null,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "SpaceXAI",
                "harness": null
            },
            "zai/glm-5.3-flash": {
                "accuracy": 71.11,
                "latency": 721.009,
                "stderr": 0.73,
                "cost_per_test": 0.017046,
                "token_totals": {
                    "input_tokens": 83673029,
                    "output_tokens": 6418255,
                    "reasoning_tokens": 4667340,
                    "cache_read_tokens": 59741440,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Zhipu AI",
                "harness": null
            },
            "xiaomi/mimo-v2.6-flash": {
                "accuracy": 70.747,
                "latency": 753.899,
                "stderr": 0.733,
                "cost_per_test": 0.020563,
                "token_totals": {
                    "input_tokens": 154083504,
                    "output_tokens": 6714632,
                    "reasoning_tokens": 4361609,
                    "cache_read_tokens": 133163328,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "xiaomi/mimo-v2.6-pro": {
                "accuracy": 69.961,
                "latency": 728.218,
                "stderr": 0.738,
                "cost_per_test": 0.050801,
                "token_totals": {
                    "input_tokens": 128656776,
                    "output_tokens": 6407181,
                    "reasoning_tokens": 4235835,
                    "cache_read_tokens": 112976384,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "Xiaomi",
                "harness": null
            },
            "minimax/MiniMax-M3": {
                "accuracy": 69.769,
                "latency": 235.083,
                "stderr": 0.74,
                "cost_per_test": 0.1087,
                "token_totals": {
                    "input_tokens": 99064650,
                    "output_tokens": 3883160,
                    "reasoning_tokens": 470716,
                    "cache_read_tokens": 86179308,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 512000,
                "reasoning": null,
                "reasoning_effort": null,
                "verbosity": null,
                "compute_effort": null,
                "provider": "MiniMax",
                "harness": null
            },
            "kimi/kimi-k3.1-pro": {
                "accuracy": 69.466,
                "latency": 384.934,
                "stderr": 0.741,
                "cost_per_test": 0.962754,
                "token_totals": {
                    "input_tokens": 350618122,
                    "output_tokens": 9004545,
                    "reasoning_tokens": 6148010,
                    "cache_read_tokens": 304815360,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "google/gemini-4-argon": {
                "accuracy": 68.485,
                "latency": 365.466,
                "stderr": 0.749,
                "cost_per_test": 0.830248,
                "token_totals": {
                    "input_tokens": 119990212,
                    "output_tokens": 5341195,
                    "reasoning_tokens": 2280691,
                    "cache_read_tokens": 99358458,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 1048576,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "kimi/kimi-k3": {
                "accuracy": 68.172,
                "latency": 582.481,
                "stderr": 0.751,
                "cost_per_test": 0.453797,
                "token_totals": {
                    "input_tokens": 49572918,
                    "output_tokens": 4976121,
                    "reasoning_tokens": 3304496,
                    "cache_read_tokens": 40371712,
                    "cache_write_tokens": 8848567
                },
                "token_tasks_measured": 252,
                "temperature": 1.0,
                "top_p": 0.95,
                "max_output_tokens": 262144,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Moonshot AI",
                "harness": null
            },
            "google/gemini-3.8-flash": {
                "accuracy": 67.057,
                "latency": 200.496,
                "stderr": 0.758,
                "cost_per_test": 0.323686,
                "token_totals": {
                    "input_tokens": 75537707,
                    "output_tokens": 4490661,
                    "reasoning_tokens": 2507217,
                    "cache_read_tokens": 48457566,
                    "cache_write_tokens": 0
                },
                "token_tasks_measured": 252,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 65536,
                "reasoning": null,
                "reasoning_effort": "high",
                "verbosity": null,
                "compute_effort": null,
                "provider": "Google",
                "harness": null
            },
            "openai/gpt-6.1-sol": {
                "accuracy": 65.836,
                "latency": 746.719,
                "stderr": 0.764,
                "cost_per_test": 0.692359,
                "token_totals": {
                    "input_tokens": 275926852,
                    "output_tokens": 7851493,
                    "reasoning_tokens": 5858261,
                    "cache_read_tokens": 247391763,
                    "cache_write_tokens": 28300248
                },
                "token_tasks_measured": 252,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            },
            "openai/gpt-5.6-luna": {
                "accuracy": 65.559,
                "latency": 316.354,
                "stderr": 0.766,
                "cost_per_test": 0.085901,
                "token_totals": {
                    "input_tokens": 252783194,
                    "output_tokens": 6867369,
                    "reasoning_tokens": 4487048,
                    "cache_read_tokens": 218606176,
                    "cache_write_tokens": 34023306
                },
                "token_tasks_measured": 252,
                "temperature": null,
                "top_p": null,
                "max_output_tokens": 128000,
                "reasoning": null,
                "reasoning_effort": "max",
                "verbosity": null,
                "compute_effort": null,
                "provider": "OpenAI",
                "harness": null
            }
        }
    }
}