Public

benredmond/apex

Updated: 8/13/2026

Languages

HTML72.5%TypeScript19.3%JavaScript7.9%Shell0.2%CSS0.1%Dockerfile<0.1%
2 Models30 Tasks

Custom repository benchmark.

Harness

1

Mini-SWE-agent
19 / 30

$0.09

6m04s

2

Mini-SWE-agent
18 / 30

$0.45

4m10s

Key Takeaways

  • GPT-5.6 Luna with Mini-SWE-agent scores 63.33% versus 60% for GPT-5.6 Terra with Mini-SWE-agent.
  • GPT-5.6 Terra with Mini-SWE-agent is faster at 249.87 seconds versus 363.72 seconds, while GPT-5.6 Luna with Mini-SWE-agent costs less per test: $0.09 versus $0.45.

Model Comparison

Accuracy

63.33%

GPT-5.6 Luna

60.00%

GPT-5.6 Terra

Task outcomes

30 tasks

Both
GPT-5.6 Luna only
GPT-5.6 Terra only
Neither
Not attempted

Cost / test

$0.09

GPT-5.6 Luna

$0.45

GPT-5.6 Terra

Cost distribution

$0.00$0.59$1.19

Latency

6m 4s

GPT-5.6 Luna

4m 10s

GPT-5.6 Terra

Latency distribution

0s8m 29s16m 58s

Cost Analysis

Cost / Test vs. Accuracy
ACCURACYCOST

Average Token Use / Test

Token Usage
InputOutputReasoningCache readCache write
GPT-5.6 Luna
2.0M
GPT-5.6 Terra
743K

Cost is the clearest tradeoff in this comparison. GPT-5.6 Luna leads at 63.33% for $0.09 per test. No other model in this comparison is cheaper.

Latency Analysis

Latency vs. Accuracy
ACCURACYLATENCY

Average Response Time / Test

Response Time
GPT-5.6 Luna
6m 4s
GPT-5.6 Terra
4m 10s

Latency separates several models with similarly strong scores. GPT-5.6 Luna leads at 63.33%, while GPT-5.6 Terra is fastest at 4m 10s with 60.00% accuracy.

Tasks with failures

Models
GPT-5.6 Luna
GPT-5.6 Terra

Task detail

87b67ad

Issue statement

Fallback pattern databases may be only partially migrated. Searching or looking up patterns with language, framework, or tag filters must not apply those facet filters to fallback databases, even if some corresponding facet or full-text-search tables happen to exist. The primary database should continue to honor all requested filters, while fallback patterns should remain eligible based on their trust/validity data and ordinary search terms without requiring matching fallback facet rows.

View Hidden Tests
diff --git a/tests/storage/repository-fallback.test.ts b/tests/storage/repository-fallback.test.tsindex f938b86..23e7bcf 100644--- a/tests/storage/repository-fallback.test.ts+++ b/tests/storage/repository-fallback.test.ts@@ -78,6 +78,124 @@ async function setupRepositoryWithFallback() {   }; } +async function setupRepositoryWithFacetFallback() {+  const previousAdapter = process.env.APEX_FORCE_ADAPTER;+  process.env.APEX_FORCE_ADAPTER = "wasm";++  const tempDir = await fs.mkdtemp(+    path.join(os.tmpdir(), "apex-repo-fallback-facets-"),+  );+  const primaryPath = path.join(tempDir, "primary.db");+  const fallbackPath = path.join(tempDir, "fallback.db");++  const fallbackDb = await DatabaseAdapterFactory.create(fallbackPath);+  fallbackDb.exec(`+    CREATE TABLE patterns (+      id TEXT PRIMARY KEY,+      schema_version TEXT NOT NULL,+      pattern_version TEXT NOT NULL,+      type TEXT NOT NULL,+      title TEXT NOT NULL,+      summary TEXT NOT NULL,+      trust_score REAL NOT NULL,+      invalid INTEGER NOT NULL DEFAULT 0,+      tags TEXT+    );+  `);+  fallbackDb.exec(`+    CREATE VIRTUAL TABLE patterns_fts USING fts3(+      id,+      title,+      summary,+      tags,+      keywords,+      search_index,+      tokenize=simple+    );+  `);+  fallbackDb.exec(`+    CREATE TABLE pattern_languages (+      pattern_id TEXT NOT NULL,+      lang TEXT NOT NULL+    );+  `);++  const fallbackPatternId = "FALLBACK:TEST:FACET:PATTERN";+  fallbackDb+    .prepare(+      `+      INSERT INTO patterns (+        rowid, id, schema_version, pattern_version, type, title, summary, trust_score, invalid, tags+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+    `,+    )+    .run(+      1,+      fallbackPatternId,+      "1.0.0",+      "1.0.0",+      "LANG",+      "Fallback Pattern With Facets",+      "authentication fallback",+      0.9,+      0,+      "[]",+    );++  fallbackDb+    .prepare(+      `+      INSERT INTO patterns_fts (+        rowid, id, title, summary, tags, keywords, search_index+      ) VALUES (?, ?, ?, ?, ?, ?, ?)+    `,+    )+    .run(+      1,+      fallbackPatternId,+      "Fallback Pattern With Facets",+      "authentication fallback",+      "[]",+      "",+      "",+    );++  fallbackDb+    .prepare(+      `+      INSERT INTO pattern_languages (pattern_id, lang) VALUES (?, ?)+    `,+    )+    .run(fallbackPatternId, "python");++  fallbackDb.close();++  const repository = await PatternRepository.create({+    dbPath: primaryPath,+    fallbackPath,+  });++  return {+    repository,+    fallbackPatternId,+    async cleanup() {+      try {+        await repository.shutdown();+      } catch (error: any) {+        if (!error || !/not open/i.test(String(error.message))) {+          throw error;+        }+      }+      await fs.remove(tempDir);+      if (previousAdapter === undefined) {+        delete process.env.APEX_FORCE_ADAPTER;+      } else {+        process.env.APEX_FORCE_ADAPTER = previousAdapter;+      }+    },+  };+}+ describe("PatternRepository fallback behavior", () => {   test("returns fallback patterns when facet tables are missing", async () => {     const { repository, cleanup, fallbackPatternId } =@@ -95,4 +213,21 @@ describe("PatternRepository fallback behavior", () => {       await cleanup();     }   });++  test("returns fallback patterns even when facet tables exist but don't match filters", async () => {+    const { repository, cleanup, fallbackPatternId } =+      await setupRepositoryWithFacetFallback();++    try {+      const results = await repository.search({+        task: "authentication",+        languages: ["typescript"],+      });++      const ids = results.patterns.map((pattern) => pattern.id);+      expect(ids).toContain(fallbackPatternId);+    } finally {+      await cleanup();+    }+  }); });