{
  "benchmark": "Semantic Grounding Benchmark",
  "version": "v2",
  "date": "2026-09-06",
  "total_model_calls": 846,
  "primary_model": "claude-haiku-4-5-20251001",
  "secondary_model": "claude-sonnet-5",
  "unit_of_analysis": "task; repeats averaged within a task before testing",
  "world": {
    "customers": 24,
    "orders": 72,
    "shipments": 66,
    "invoices": 56,
    "returns": 10,
    "organisations": 3
  },
  "track_A_grounding": {
    "arms": {
      "schema": {
        "answer_accuracy": 0.2647,
        "definition_cited": 0.0,
        "correct_abstention": 0.8333,
        "competing_answer": 0.1618,
        "prompt_chars": 11176
      },
      "prose": {
        "answer_accuracy": 0.2941,
        "definition_cited": 0.0,
        "correct_abstention": 0.8333,
        "competing_answer": 0.3529,
        "prompt_chars": 12569
      },
      "prose_full": {
        "answer_accuracy": 0.7647,
        "definition_cited": 0.0,
        "correct_abstention": 0.6667,
        "competing_answer": 0.0294,
        "prompt_chars": 14007
      },
      "vercy": {
        "answer_accuracy": 0.7647,
        "definition_cited": 1.0,
        "correct_abstention": 0.5833,
        "competing_answer": 0.0147,
        "prompt_chars": 15079
      },
      "vercy_pad": {
        "answer_accuracy": 0.8529,
        "definition_cited": 1.0,
        "correct_abstention": 0.6667,
        "competing_answer": 0.0,
        "prompt_chars": 16486
      },
      "vercy_fed": {
        "answer_accuracy": 0.7794,
        "definition_cited": 0.9853,
        "correct_abstention": 0.6667,
        "competing_answer": 0.0147,
        "prompt_chars": 16816
      }
    },
    "by_family": {
      "schema": {
        "COMP": 0.4167,
        "TEMP": 0.2,
        "XORG": 0.1667,
        "ABST": 0.8333
      },
      "prose": {
        "COMP": 0.375,
        "TEMP": 0.5,
        "XORG": 0.0417,
        "ABST": 0.8333
      },
      "prose_full": {
        "COMP": 0.7917,
        "TEMP": 0.95,
        "XORG": 0.5833,
        "ABST": 0.6667
      },
      "vercy": {
        "COMP": 0.8333,
        "TEMP": 0.9,
        "XORG": 0.5833,
        "ABST": 0.5833
      },
      "vercy_pad": {
        "COMP": 0.875,
        "TEMP": 0.9,
        "XORG": 0.7917,
        "ABST": 0.6667
      },
      "vercy_fed": {
        "COMP": 0.7917,
        "TEMP": 0.85,
        "XORG": 0.7083,
        "ABST": 0.6667
      }
    },
    "paired_sign_test_p": {
      "schema_vs_vercy": 0.000546,
      "prose_vs_vercy": 0.001514,
      "prose_full_vs_vercy": 1.0,
      "vercy_vs_vercy_pad": 0.226562,
      "vercy_vs_vercy_fed": 1.0,
      "schema_vs_prose_full": 0.001514
    },
    "sonnet_on_v1_instrument": {
      "schema": {
        "answer_accuracy": 0.2059,
        "competing_answer": 0.1765,
        "correct_abstention": 1.0
      },
      "prose": {
        "answer_accuracy": 0.2647,
        "competing_answer": 0.1176,
        "correct_abstention": 1.0
      },
      "vercy": {
        "answer_accuracy": 0.7941,
        "competing_answer": 0.0,
        "correct_abstention": 0.8333
      },
      "vercy_fed": {
        "answer_accuracy": 0.7353,
        "competing_answer": 0.0,
        "correct_abstention": 0.8333
      }
    }
  },
  "track_B1_elicit": {
    "schema": {
      "gap_found": 0.0,
      "field_named": 0.0,
      "owner_named": 0.0,
      "false_alarm": 0.0,
      "control_correct": 0.125,
      "gaps": 12,
      "controls": 8
    },
    "prose_deficient": {
      "gap_found": 0.0,
      "field_named": 0.0,
      "owner_named": 0.0,
      "false_alarm": 0.0,
      "control_correct": 0.5,
      "gaps": 12,
      "controls": 8
    },
    "vercy_deficient": {
      "gap_found": 0.8333,
      "field_named": 0.8333,
      "owner_named": 0.8333,
      "false_alarm": 0.0,
      "control_correct": 0.875,
      "gaps": 12,
      "controls": 8
    }
  },
  "track_B2_assemble": {
    "catalogue_records": 41,
    "selection_recall": 1.0,
    "selection_precision": 0.95,
    "records_selected_mean": 1.2,
    "full_catalogue": {
      "accuracy": 0.8,
      "context_chars": 10332,
      "n": 20
    },
    "self_selected": {
      "accuracy": 0.6,
      "context_chars": 472,
      "n": 20
    },
    "keyword_top5": {
      "accuracy": 0.6,
      "context_chars": 1988,
      "n": 20
    }
  },
  "track_B3_exchange": {
    "no_policy": {
      "precision": 0.3913,
      "recall": 0.45,
      "accuracy": 0.6528,
      "critical_leaks": 5,
      "decisions": 72
    },
    "prose_policy": {
      "precision": 0.6667,
      "recall": 1.0,
      "accuracy": 0.8611,
      "critical_leaks": 2,
      "decisions": 72
    },
    "vercy_policy": {
      "precision": 1.0,
      "recall": 1.0,
      "accuracy": 1.0,
      "critical_leaks": 0,
      "decisions": 72
    }
  },
  "track_B4_collab": {
    "prose_deficient": {
      "routing_correct": 0.5,
      "record_well_formed": 1.0,
      "effective_date_right": 0.75,
      "final_answer_correct": 0.5,
      "chains": 8
    },
    "vercy_deficient": {
      "routing_correct": 1.0,
      "record_well_formed": 1.0,
      "effective_date_right": 0.75,
      "final_answer_correct": 0.5,
      "chains": 8
    }
  },
  "headline": {
    "accuracy_no_definitions": 0.2647,
    "accuracy_incomplete_prose": 0.2941,
    "accuracy_complete_prose": 0.7647,
    "accuracy_structured": 0.7647,
    "structure_vs_complete_prose_p": 1.0,
    "traceability_complete_prose": 0.0,
    "traceability_structured": 1.0,
    "semantic_error_no_docs": 0.1618,
    "semantic_error_incomplete_prose": 0.3529,
    "gap_detection_prose": 0.0,
    "gap_detection_structured": 0.8333,
    "critical_leaks_prose": 2,
    "critical_leaks_structured": 0,
    "routing_prose": 0.5,
    "routing_structured": 1.0,
    "context_reduction": 0.9543
  }
}