{
  "benchmark": "vercy-memory-retrieval-v1",
  "version": "1.0.0",
  "runAt": "2026-09-05T21:09:51Z",
  "provider": "Claude Code CLI",
  "requestedModel": "sonnet",
  "trials": 3,
  "conditions": {
    "vercy-retrieval": {
      "contextSha256": "97c05a4c173de8a4539fd0743c9fecd55c6ea70362dc68fd380c6a3a3d95fa18",
      "contextCharacters": 43456,
      "runs": [
        {
          "trial": 1,
          "metadata": {
            "subtype": "success",
            "duration_ms": 7354,
            "duration_api_ms": 7299,
            "num_turns": 1,
            "total_cost_usd": 0.1492617,
            "modelUsage": {
              "claude-sonnet-5": {
                "inputTokens": 2,
                "outputTokens": 776,
                "cacheReadInputTokens": 9659,
                "cacheCreationInputTokens": 22453,
                "webSearchRequests": 0,
                "costUSD": 0.1492617,
                "contextWindow": 1000000,
                "maxOutputTokens": 64000,
                "canonicalModel": "claude-sonnet-5",
                "provider": "firstParty"
              }
            }
          },
          "score": {
            "correct": 15,
            "total": 15,
            "details": [
              {
                "id": "Q01",
                "answer": "09:00-17:00",
                "correct": true
              },
              {
                "id": "Q02",
                "answer": "10:00-19:00",
                "correct": true
              },
              {
                "id": "Q03",
                "answer": "maintenance",
                "correct": true
              },
              {
                "id": "Q04",
                "answer": "ready",
                "correct": true
              },
              {
                "id": "Q05",
                "answer": "80 W",
                "correct": true
              },
              {
                "id": "Q06",
                "answer": "Omar Silva",
                "correct": true
              },
              {
                "id": "Q07",
                "answer": "Fabrication Lab",
                "correct": true
              },
              {
                "id": "Q08",
                "answer": "Wayfinding Kiosk",
                "correct": true
              },
              {
                "id": "Q09",
                "answer": "24 sheets",
                "correct": true
              },
              {
                "id": "Q10",
                "answer": "2.1",
                "correct": true
              },
              {
                "id": "Q11",
                "answer": "2026-09-15",
                "correct": true
              },
              {
                "id": "Q12",
                "answer": "Contested: 30 cm (per manual, asserted) vs 45 cm (per inspection, contested status)",
                "correct": true
              },
              {
                "id": "Q13",
                "answer": "Northstar Laser Cutter LC-80",
                "correct": true
              },
              {
                "id": "Q14",
                "answer": "Maya Chen",
                "correct": true
              },
              {
                "id": "Q15",
                "answer": "WO-104",
                "correct": true
              }
            ]
          },
          "rawAnswer": "```json\n{\"answers\":[{\"id\":\"Q01\",\"answer\":\"09:00-17:00\"},{\"id\":\"Q02\",\"answer\":\"10:00-19:00\"},{\"id\":\"Q03\",\"answer\":\"maintenance\"},{\"id\":\"Q04\",\"answer\":\"ready\"},{\"id\":\"Q05\",\"answer\":\"80 W\"},{\"id\":\"Q06\",\"answer\":\"Omar Silva\"},{\"id\":\"Q07\",\"answer\":\"Fabrication Lab\"},{\"id\":\"Q08\",\"answer\":\"Wayfinding Kiosk\"},{\"id\":\"Q09\",\"answer\":\"24 sheets\"},{\"id\":\"Q10\",\"answer\":\"2.1\"},{\"id\":\"Q11\",\"answer\":\"2026-09-15\"},{\"id\":\"Q12\",\"answer\":\"Contested: 30 cm (per manual, asserted) vs 45 cm (per inspection, contested status)\"},{\"id\":\"Q13\",\"answer\":\"Northstar Laser Cutter LC-80\"},{\"id\":\"Q14\",\"answer\":\"Maya Chen\"},{\"id\":\"Q15\",\"answer\":\"WO-104\"}]}\n```"
        },
        {
          "trial": 2,
          "metadata": {
            "subtype": "success",
            "duration_ms": 7543,
            "duration_api_ms": 7486,
            "num_turns": 1,
            "total_cost_usd": 0.021939600000000004,
            "modelUsage": {
              "claude-sonnet-5": {
                "inputTokens": 2,
                "outputTokens": 820,
                "cacheReadInputTokens": 32112,
                "cacheCreationInputTokens": 0,
                "webSearchRequests": 0,
                "costUSD": 0.021939600000000004,
                "contextWindow": 1000000,
                "maxOutputTokens": 64000,
                "canonicalModel": "claude-sonnet-5",
                "provider": "firstParty"
              }
            }
          },
          "score": {
            "correct": 15,
            "total": 15,
            "details": [
              {
                "id": "Q01",
                "answer": "09:00-17:00",
                "correct": true
              },
              {
                "id": "Q02",
                "answer": "10:00-19:00",
                "correct": true
              },
              {
                "id": "Q03",
                "answer": "maintenance",
                "correct": true
              },
              {
                "id": "Q04",
                "answer": "ready",
                "correct": true
              },
              {
                "id": "Q05",
                "answer": "80 W",
                "correct": true
              },
              {
                "id": "Q06",
                "answer": "Omar Silva",
                "correct": true
              },
              {
                "id": "Q07",
                "answer": "Fabrication Lab",
                "correct": true
              },
              {
                "id": "Q08",
                "answer": "Wayfinding Kiosk",
                "correct": true
              },
              {
                "id": "Q09",
                "answer": "24 sheets",
                "correct": true
              },
              {
                "id": "Q10",
                "answer": "2.1",
                "correct": true
              },
              {
                "id": "Q11",
                "answer": "2026-09-15",
                "correct": true
              },
              {
                "id": "Q12",
                "answer": "Contested: 30 cm (manual, confidence 1.0) vs 45 cm (inspection, confidence 0.8, status contested)",
                "correct": true
              },
              {
                "id": "Q13",
                "answer": "Northstar Laser Cutter LC-80",
                "correct": true
              },
              {
                "id": "Q14",
                "answer": "Maya Chen",
                "correct": true
              },
              {
                "id": "Q15",
                "answer": "WO-104",
                "correct": true
              }
            ]
          },
          "rawAnswer": "{\"answers\":[{\"id\":\"Q01\",\"answer\":\"09:00-17:00\"},{\"id\":\"Q02\",\"answer\":\"10:00-19:00\"},{\"id\":\"Q03\",\"answer\":\"maintenance\"},{\"id\":\"Q04\",\"answer\":\"ready\"},{\"id\":\"Q05\",\"answer\":\"80 W\"},{\"id\":\"Q06\",\"answer\":\"Omar Silva\"},{\"id\":\"Q07\",\"answer\":\"Fabrication Lab\"},{\"id\":\"Q08\",\"answer\":\"Wayfinding Kiosk\"},{\"id\":\"Q09\",\"answer\":\"24 sheets\"},{\"id\":\"Q10\",\"answer\":\"2.1\"},{\"id\":\"Q11\",\"answer\":\"2026-09-15\"},{\"id\":\"Q12\",\"answer\":\"Contested: 30 cm (manual, confidence 1.0) vs 45 cm (inspection, confidence 0.8, status contested)\"},{\"id\":\"Q13\",\"answer\":\"Northstar Laser Cutter LC-80\"},{\"id\":\"Q14\",\"answer\":\"Maya Chen\"},{\"id\":\"Q15\",\"answer\":\"WO-104\"}]}"
        },
        {
          "trial": 3,
          "metadata": {
            "subtype": "success",
            "duration_ms": 9481,
            "duration_api_ms": 9426,
            "num_turns": 1,
            "total_cost_usd": 0.023709600000000004,
            "modelUsage": {
              "claude-sonnet-5": {
                "inputTokens": 2,
                "outputTokens": 938,
                "cacheReadInputTokens": 32112,
                "cacheCreationInputTokens": 0,
                "webSearchRequests": 0,
                "costUSD": 0.023709600000000004,
                "contextWindow": 1000000,
                "maxOutputTokens": 64000,
                "canonicalModel": "claude-sonnet-5",
                "provider": "firstParty"
              }
            }
          },
          "score": {
            "correct": 15,
            "total": 15,
            "details": [
              {
                "id": "Q01",
                "answer": "09:00-17:00",
                "correct": true
              },
              {
                "id": "Q02",
                "answer": "10:00-19:00",
                "correct": true
              },
              {
                "id": "Q03",
                "answer": "maintenance",
                "correct": true
              },
              {
                "id": "Q04",
                "answer": "ready",
                "correct": true
              },
              {
                "id": "Q05",
                "answer": "80 W",
                "correct": true
              },
              {
                "id": "Q06",
                "answer": "Omar Silva",
                "correct": true
              },
              {
                "id": "Q07",
                "answer": "Fabrication Lab",
                "correct": true
              },
              {
                "id": "Q08",
                "answer": "Wayfinding Kiosk",
                "correct": true
              },
              {
                "id": "Q09",
                "answer": "24 sheets",
                "correct": true
              },
              {
                "id": "Q10",
                "answer": "2.1",
                "correct": true
              },
              {
                "id": "Q11",
                "answer": "2026-09-15",
                "correct": true
              },
              {
                "id": "Q12",
                "answer": "Contested: 30 cm (manual, asserted) vs 45 cm (inspection, contested, confidence 0.8), both valid from 2026-07-01",
                "correct": true
              },
              {
                "id": "Q13",
                "answer": "Northstar Laser Cutter LC-80",
                "correct": true
              },
              {
                "id": "Q14",
                "answer": "Maya Chen",
                "correct": true
              },
              {
                "id": "Q15",
                "answer": "WO-104",
                "correct": true
              }
            ]
          },
          "rawAnswer": "{\"answers\":[{\"id\":\"Q01\",\"answer\":\"09:00-17:00\"},{\"id\":\"Q02\",\"answer\":\"10:00-19:00\"},{\"id\":\"Q03\",\"answer\":\"maintenance\"},{\"id\":\"Q04\",\"answer\":\"ready\"},{\"id\":\"Q05\",\"answer\":\"80 W\"},{\"id\":\"Q06\",\"answer\":\"Omar Silva\"},{\"id\":\"Q07\",\"answer\":\"Fabrication Lab\"},{\"id\":\"Q08\",\"answer\":\"Wayfinding Kiosk\"},{\"id\":\"Q09\",\"answer\":\"24 sheets\"},{\"id\":\"Q10\",\"answer\":\"2.1\"},{\"id\":\"Q11\",\"answer\":\"2026-09-15\"},{\"id\":\"Q12\",\"answer\":\"Contested: 30 cm (manual, asserted) vs 45 cm (inspection, contested, confidence 0.8), both valid from 2026-07-01\"},{\"id\":\"Q13\",\"answer\":\"Northstar Laser Cutter LC-80\"},{\"id\":\"Q14\",\"answer\":\"Maya Chen\"},{\"id\":\"Q15\",\"answer\":\"WO-104\"}]}"
        }
      ],
      "meanAccuracy": 1.0
    },
    "flat-notes": {
      "contextSha256": "e3978771190019f61a082cf10108788f6d8515683531f3152bbd23c94733f67f",
      "contextCharacters": 103213,
      "runs": [
        {
          "trial": 1,
          "metadata": {
            "subtype": "success",
            "duration_ms": 10595,
            "duration_api_ms": 10534,
            "num_turns": 1,
            "total_cost_usd": 0.3209727,
            "modelUsage": {
              "claude-sonnet-5": {
                "inputTokens": 2,
                "outputTokens": 1043,
                "cacheReadInputTokens": 9659,
                "cacheCreationInputTokens": 50404,
                "webSearchRequests": 0,
                "costUSD": 0.3209727,
                "contextWindow": 1000000,
                "maxOutputTokens": 64000,
                "canonicalModel": "claude-sonnet-5",
                "provider": "firstParty"
              }
            }
          },
          "score": {
            "correct": 15,
            "total": 15,
            "details": [
              {
                "id": "Q01",
                "answer": "09:00-17:00",
                "correct": true
              },
              {
                "id": "Q02",
                "answer": "10:00-19:00",
                "correct": true
              },
              {
                "id": "Q03",
                "answer": "maintenance",
                "correct": true
              },
              {
                "id": "Q04",
                "answer": "ready",
                "correct": true
              },
              {
                "id": "Q05",
                "answer": "80 W",
                "correct": true
              },
              {
                "id": "Q06",
                "answer": "Omar Silva (northstar.omar)",
                "correct": true
              },
              {
                "id": "Q07",
                "answer": "Fabrication Lab (northstar.lab)",
                "correct": true
              },
              {
                "id": "Q08",
                "answer": "Wayfinding Kiosk (northstar.kiosk)",
                "correct": true
              },
              {
                "id": "Q09",
                "answer": "24 sheets",
                "correct": true
              },
              {
                "id": "Q10",
                "answer": "2.1",
                "correct": true
              },
              {
                "id": "Q11",
                "answer": "2026-09-15",
                "correct": true
              },
              {
                "id": "Q12",
                "answer": "Contested: 30 cm (asserted, source northstar.synthetic.manual) vs 45 cm (contested, source northstar.synthetic.inspection), both valid from 2026-07-01",
                "correct": true
              },
              {
                "id": "Q13",
                "answer": "Northstar Laser Cutter LC-80",
                "correct": true
              },
              {
                "id": "Q14",
                "answer": "Maya Chen (northstar.maya)",
                "correct": true
              },
              {
                "id": "Q15",
                "answer": "WO-104",
                "correct": true
              }
            ]
          },
          "rawAnswer": "{\"answers\":[{\"id\":\"Q01\",\"answer\":\"09:00-17:00\"},{\"id\":\"Q02\",\"answer\":\"10:00-19:00\"},{\"id\":\"Q03\",\"answer\":\"maintenance\"},{\"id\":\"Q04\",\"answer\":\"ready\"},{\"id\":\"Q05\",\"answer\":\"80 W\"},{\"id\":\"Q06\",\"answer\":\"Omar Silva (northstar.omar)\"},{\"id\":\"Q07\",\"answer\":\"Fabrication Lab (northstar.lab)\"},{\"id\":\"Q08\",\"answer\":\"Wayfinding Kiosk (northstar.kiosk)\"},{\"id\":\"Q09\",\"answer\":\"24 sheets\"},{\"id\":\"Q10\",\"answer\":\"2.1\"},{\"id\":\"Q11\",\"answer\":\"2026-09-15\"},{\"id\":\"Q12\",\"answer\":\"Contested: 30 cm (asserted, source northstar.synthetic.manual) vs 45 cm (contested, source northstar.synthetic.inspection), both valid from 2026-07-01\"},{\"id\":\"Q13\",\"answer\":\"Northstar Laser Cutter LC-80\"},{\"id\":\"Q14\",\"answer\":\"Maya Chen (northstar.maya)\"},{\"id\":\"Q15\",\"answer\":\"WO-104\"}]}"
        },
        {
          "trial": 2,
          "metadata": {
            "subtype": "success",
            "duration_ms": 9193,
            "duration_api_ms": 9136,
            "num_turns": 1,
            "total_cost_usd": 0.0317649,
            "modelUsage": {
              "claude-sonnet-5": {
                "inputTokens": 2,
                "outputTokens": 916,
                "cacheReadInputTokens": 60063,
                "cacheCreationInputTokens": 0,
                "webSearchRequests": 0,
                "costUSD": 0.0317649,
                "contextWindow": 1000000,
                "maxOutputTokens": 64000,
                "canonicalModel": "claude-sonnet-5",
                "provider": "firstParty"
              }
            }
          },
          "score": {
            "correct": 15,
            "total": 15,
            "details": [
              {
                "id": "Q01",
                "answer": "09:00-17:00",
                "correct": true
              },
              {
                "id": "Q02",
                "answer": "10:00-19:00",
                "correct": true
              },
              {
                "id": "Q03",
                "answer": "maintenance",
                "correct": true
              },
              {
                "id": "Q04",
                "answer": "ready",
                "correct": true
              },
              {
                "id": "Q05",
                "answer": "80 W",
                "correct": true
              },
              {
                "id": "Q06",
                "answer": "Omar Silva",
                "correct": true
              },
              {
                "id": "Q07",
                "answer": "Fabrication Lab",
                "correct": true
              },
              {
                "id": "Q08",
                "answer": "Wayfinding Kiosk",
                "correct": true
              },
              {
                "id": "Q09",
                "answer": "24 sheets",
                "correct": true
              },
              {
                "id": "Q10",
                "answer": "2.1",
                "correct": true
              },
              {
                "id": "Q11",
                "answer": "2026-09-15",
                "correct": true
              },
              {
                "id": "Q12",
                "answer": "Contested: 30 cm (asserted, source northstar.synthetic.manual) vs 45 cm (contested, source northstar.synthetic.inspection), both valid from 2026-07-01",
                "correct": true
              },
              {
                "id": "Q13",
                "answer": "Northstar Laser Cutter LC-80",
                "correct": true
              },
              {
                "id": "Q14",
                "answer": "Maya Chen",
                "correct": true
              },
              {
                "id": "Q15",
                "answer": "WO-104",
                "correct": true
              }
            ]
          },
          "rawAnswer": "{\"answers\":[{\"id\":\"Q01\",\"answer\":\"09:00-17:00\"},{\"id\":\"Q02\",\"answer\":\"10:00-19:00\"},{\"id\":\"Q03\",\"answer\":\"maintenance\"},{\"id\":\"Q04\",\"answer\":\"ready\"},{\"id\":\"Q05\",\"answer\":\"80 W\"},{\"id\":\"Q06\",\"answer\":\"Omar Silva\"},{\"id\":\"Q07\",\"answer\":\"Fabrication Lab\"},{\"id\":\"Q08\",\"answer\":\"Wayfinding Kiosk\"},{\"id\":\"Q09\",\"answer\":\"24 sheets\"},{\"id\":\"Q10\",\"answer\":\"2.1\"},{\"id\":\"Q11\",\"answer\":\"2026-09-15\"},{\"id\":\"Q12\",\"answer\":\"Contested: 30 cm (asserted, source northstar.synthetic.manual) vs 45 cm (contested, source northstar.synthetic.inspection), both valid from 2026-07-01\"},{\"id\":\"Q13\",\"answer\":\"Northstar Laser Cutter LC-80\"},{\"id\":\"Q14\",\"answer\":\"Maya Chen\"},{\"id\":\"Q15\",\"answer\":\"WO-104\"}]}"
        },
        {
          "trial": 3,
          "metadata": {
            "subtype": "success",
            "duration_ms": 10812,
            "duration_api_ms": 10756,
            "num_turns": 1,
            "total_cost_usd": 0.033024899999999996,
            "modelUsage": {
              "claude-sonnet-5": {
                "inputTokens": 2,
                "outputTokens": 1000,
                "cacheReadInputTokens": 60063,
                "cacheCreationInputTokens": 0,
                "webSearchRequests": 0,
                "costUSD": 0.033024899999999996,
                "contextWindow": 1000000,
                "maxOutputTokens": 64000,
                "canonicalModel": "claude-sonnet-5",
                "provider": "firstParty"
              }
            }
          },
          "score": {
            "correct": 15,
            "total": 15,
            "details": [
              {
                "id": "Q01",
                "answer": "09:00-17:00",
                "correct": true
              },
              {
                "id": "Q02",
                "answer": "10:00-19:00",
                "correct": true
              },
              {
                "id": "Q03",
                "answer": "maintenance",
                "correct": true
              },
              {
                "id": "Q04",
                "answer": "ready",
                "correct": true
              },
              {
                "id": "Q05",
                "answer": "80 W",
                "correct": true
              },
              {
                "id": "Q06",
                "answer": "Omar Silva",
                "correct": true
              },
              {
                "id": "Q07",
                "answer": "Fabrication Lab",
                "correct": true
              },
              {
                "id": "Q08",
                "answer": "Wayfinding Kiosk",
                "correct": true
              },
              {
                "id": "Q09",
                "answer": "24 sheets",
                "correct": true
              },
              {
                "id": "Q10",
                "answer": "2.1",
                "correct": true
              },
              {
                "id": "Q11",
                "answer": "2026-09-15",
                "correct": true
              },
              {
                "id": "Q12",
                "answer": "Contested: 30 cm (source: operations, status asserted) vs 45 cm (source: inspection, status contested), both valid from 2026-07-01 to open",
                "correct": true
              },
              {
                "id": "Q13",
                "answer": "Northstar Laser Cutter LC-80",
                "correct": true
              },
              {
                "id": "Q14",
                "answer": "Maya Chen",
                "correct": true
              },
              {
                "id": "Q15",
                "answer": "WO-104",
                "correct": true
              }
            ]
          },
          "rawAnswer": "{\"answers\":[{\"id\":\"Q01\",\"answer\":\"09:00-17:00\"},{\"id\":\"Q02\",\"answer\":\"10:00-19:00\"},{\"id\":\"Q03\",\"answer\":\"maintenance\"},{\"id\":\"Q04\",\"answer\":\"ready\"},{\"id\":\"Q05\",\"answer\":\"80 W\"},{\"id\":\"Q06\",\"answer\":\"Omar Silva\"},{\"id\":\"Q07\",\"answer\":\"Fabrication Lab\"},{\"id\":\"Q08\",\"answer\":\"Wayfinding Kiosk\"},{\"id\":\"Q09\",\"answer\":\"24 sheets\"},{\"id\":\"Q10\",\"answer\":\"2.1\"},{\"id\":\"Q11\",\"answer\":\"2026-09-15\"},{\"id\":\"Q12\",\"answer\":\"Contested: 30 cm (source: operations, status asserted) vs 45 cm (source: inspection, status contested), both valid from 2026-07-01 to open\"},{\"id\":\"Q13\",\"answer\":\"Northstar Laser Cutter LC-80\"},{\"id\":\"Q14\",\"answer\":\"Maya Chen\"},{\"id\":\"Q15\",\"answer\":\"WO-104\"}]}"
        }
      ],
      "meanAccuracy": 1.0
    }
  }
}