{
  "benchmark": "Collaborative Memory Benchmark",
  "version": "v1",
  "date": "2026-09-07",
  "model": "claude-haiku-4-5-20251001",
  "unit_of_analysis": "item; replicates averaged within an item before testing",
  "total_model_calls": 510,
  "calls": {
    "grid": 420,
    "scope": 60,
    "restatement": 30
  },
  "organisation": {
    "teams": 6,
    "records": 30,
    "concepts": 8,
    "restricted_records": 5
  },
  "items": {
    "total": 35,
    "answerable": 31,
    "families": {
      "AUTH": 6,
      "CONF": 6,
      "HIST": 5,
      "STALE": 5,
      "ROUTE": 5,
      "PROV": 4,
      "GAP": 4
    }
  },
  "arms": {
    "wiki": {
      "answer_accuracy": 0.9677,
      "ci95": [
        0.9238,
        1.0
      ],
      "owner_named": 1.0,
      "policy_named": 1.0,
      "source_cited": 1.0,
      "correct_abstention": 1.0,
      "false_abstention": 0.0,
      "superseded_judgement": 0.8,
      "rival_team_value": 0.0,
      "fabrication": 0.0,
      "context_chars": 6618
    },
    "vector_memory": {
      "answer_accuracy": 0.5645,
      "ci95": [
        0.4023,
        0.7267
      ],
      "owner_named": 0.7708,
      "policy_named": 0.0,
      "source_cited": 0.0,
      "correct_abstention": 1.0,
      "false_abstention": 0.129,
      "superseded_judgement": 1.0,
      "rival_team_value": 0.0323,
      "fabrication": 0.0484,
      "context_chars": 2694
    },
    "bitemporal": {
      "answer_accuracy": 0.7742,
      "ci95": [
        0.6553,
        0.893
      ],
      "owner_named": 0.7917,
      "policy_named": 0.0,
      "source_cited": 1.0,
      "correct_abstention": 1.0,
      "false_abstention": 0.0806,
      "superseded_judgement": 0.8,
      "rival_team_value": 0.0484,
      "fabrication": 0.0161,
      "context_chars": 8932
    },
    "graph_catalog": {
      "answer_accuracy": 0.9355,
      "ci95": [
        0.8602,
        1.0
      ],
      "owner_named": 1.0,
      "policy_named": 1.0,
      "source_cited": 1.0,
      "correct_abstention": 1.0,
      "false_abstention": 0.0,
      "superseded_judgement": 0.9,
      "rival_team_value": 0.0,
      "fabrication": 0.0,
      "context_chars": 10621
    },
    "dimension": {
      "answer_accuracy": 0.9516,
      "ci95": [
        0.8819,
        1.0
      ],
      "owner_named": 1.0,
      "policy_named": 1.0,
      "source_cited": 1.0,
      "correct_abstention": 1.0,
      "false_abstention": 0.0,
      "superseded_judgement": 1.0,
      "rival_team_value": 0.0,
      "fabrication": 0.0,
      "context_chars": 11191
    },
    "dim_no_gov": {
      "answer_accuracy": 0.7419,
      "ci95": [
        0.5921,
        0.8918
      ],
      "owner_named": 0.6875,
      "policy_named": 0.0,
      "source_cited": 1.0,
      "correct_abstention": 1.0,
      "false_abstention": 0.0968,
      "superseded_judgement": 0.8,
      "rival_team_value": 0.0645,
      "fabrication": 0.0161,
      "context_chars": 6474
    }
  },
  "by_family": {
    "wiki": {
      "AUTH": 1.0,
      "CONF": 0.9167,
      "HIST": 1.0,
      "STALE": 1.0,
      "ROUTE": 1.0,
      "PROV": 0.875,
      "GAP": 1.0
    },
    "vector_memory": {
      "AUTH": 0.6667,
      "CONF": 0.3333,
      "HIST": 0.7,
      "STALE": 0.7,
      "ROUTE": 0.9,
      "PROV": 0.0,
      "GAP": 1.0
    },
    "bitemporal": {
      "AUTH": 0.75,
      "CONF": 0.4167,
      "HIST": 0.9,
      "STALE": 0.8,
      "ROUTE": 0.9,
      "PROV": 1.0,
      "GAP": 1.0
    },
    "graph_catalog": {
      "AUTH": 1.0,
      "CONF": 0.9167,
      "HIST": 1.0,
      "STALE": 0.9,
      "ROUTE": 1.0,
      "PROV": 0.75,
      "GAP": 1.0
    },
    "dimension": {
      "AUTH": 1.0,
      "CONF": 1.0,
      "HIST": 1.0,
      "STALE": 0.7,
      "ROUTE": 1.0,
      "PROV": 1.0,
      "GAP": 1.0
    },
    "dim_no_gov": {
      "AUTH": 0.8333,
      "CONF": 0.3333,
      "HIST": 1.0,
      "STALE": 0.6,
      "ROUTE": 0.8,
      "PROV": 1.0,
      "GAP": 1.0
    }
  },
  "paired_sign_tests": {
    "wiki_vs_dimension": {
      "p": 1.0,
      "better": 2,
      "worse": 2,
      "a": 0.9677,
      "b": 0.9516
    },
    "vector_memory_vs_dimension": {
      "p": 0.00012,
      "better": 14,
      "worse": 0,
      "a": 0.5645,
      "b": 0.9516
    },
    "bitemporal_vs_dimension": {
      "p": 0.02246,
      "better": 11,
      "worse": 2,
      "a": 0.7742,
      "b": 0.9516
    },
    "graph_catalog_vs_dimension": {
      "p": 1.0,
      "better": 2,
      "worse": 2,
      "a": 0.9355,
      "b": 0.9516
    },
    "dim_no_gov_vs_dimension": {
      "p": 0.01562,
      "better": 7,
      "worse": 0,
      "a": 0.7419,
      "b": 0.9516
    },
    "bitemporal_vs_graph_catalog": {
      "p": 0.03857,
      "better": 10,
      "worse": 2,
      "a": 0.7742,
      "b": 0.9355
    },
    "bitemporal_vs_dim_no_gov": {
      "p": 1.0,
      "better": 5,
      "worse": 6,
      "a": 0.7742,
      "b": 0.7419
    },
    "vector_memory_vs_bitemporal": {
      "p": 0.05737,
      "better": 11,
      "worse": 3,
      "a": 0.5645,
      "b": 0.7742
    },
    "wiki_vs_graph_catalog": {
      "p": 0.625,
      "better": 1,
      "worse": 3,
      "a": 0.9677,
      "b": 0.9355
    }
  },
  "governance_ablation": {
    "AUTH": {
      "ablated": 0.8333,
      "full": 1.0,
      "p": 1.0,
      "role": "target"
    },
    "CONF": {
      "ablated": 0.3333,
      "full": 1.0,
      "p": 0.125,
      "role": "target"
    },
    "ROUTE": {
      "ablated": 0.8,
      "full": 1.0,
      "p": 1.0,
      "role": "target"
    },
    "HIST": {
      "ablated": 1.0,
      "full": 1.0,
      "p": 1.0,
      "role": "control"
    }
  },
  "fidelity_check": {
    "bitemporal": 0.7742,
    "dim_no_gov": 0.7419,
    "p": 1.0
  },
  "scope": {
    "wiki": {
      "precision": 1.0,
      "recall": 1.0,
      "accuracy": 1.0,
      "critical_over_shares": 0,
      "decisions": 60
    },
    "vector_memory": {
      "precision": 0.9524,
      "recall": 0.9091,
      "accuracy": 0.95,
      "critical_over_shares": 1,
      "decisions": 60
    },
    "bitemporal": {
      "precision": 0.8667,
      "recall": 0.5909,
      "accuracy": 0.8167,
      "critical_over_shares": 2,
      "decisions": 60
    },
    "graph_catalog": {
      "precision": 1.0,
      "recall": 1.0,
      "accuracy": 1.0,
      "critical_over_shares": 0,
      "decisions": 60
    },
    "dimension": {
      "precision": 1.0,
      "recall": 1.0,
      "accuracy": 1.0,
      "critical_over_shares": 0,
      "decisions": 60
    }
  },
  "restatement": {
    "wiki": {
      "value_ok": 1.0,
      "from_ok": 0.8,
      "team_ok": 1.0,
      "close_ok": 0.0,
      "history_ok": 1.0,
      "other_ok": 1.0,
      "well_formed": 0.0,
      "chains": 10
    },
    "bitemporal": {
      "value_ok": 1.0,
      "from_ok": 0.8,
      "team_ok": 1.0,
      "close_ok": 1.0,
      "history_ok": 1.0,
      "other_ok": 1.0,
      "well_formed": 0.8,
      "chains": 10
    },
    "dimension": {
      "value_ok": 1.0,
      "from_ok": 0.8,
      "team_ok": 1.0,
      "close_ok": 1.0,
      "history_ok": 1.0,
      "other_ok": 1.0,
      "well_formed": 0.8,
      "chains": 10
    }
  }
}