# Vercy AI instruction - YAML 1.2 (JSON-compatible) { "vercy": "1.0-draft", "publication": { "status": "published", "adjudicationStatus": "reviewable-draft", "publishableCanonical": false, "generatedAt": "2026-10-21T16:57:27Z", "synthesisSha256": "682b2427376bac5b581da4a72b69517cc879dea4bcce07ff05beef23b2243bf5", "providerMode": "single-provider-waiver", "providers": [ "Codex" ], "waivedProviders": [ "Claude", "Grok" ] }, "metaModel": { "id": "WM-AI-011", "registryId": "vr.wm-ai-011", "name": "Evaluation Harness and Scorer", "version": "0.1.0", "previousVersions": [], "entryKind": "entity", "family": "World Models", "category": "Information and virtual systems", "industry": [ "Cross-industry" ], "domain": [ "INF.AI.HRN" ], "tags": [ "evaluation", "harness", "scorer", "inf.ai.hrn" ], "status": "published" }, "canonicalUrl": "https://ver.cy/models/wm-ai-011-evaluation-harness-and-scorer/", "sourceUrl": "https://github.com/ver-cy/world-models/tree/feat/mega-model-registry/research/runs/wm-ai-011", "model": { "registry_id": "vr.wm-ai-011", "model_id": "WM-AI-011", "name": "Evaluation Harness and Scorer", "entry_kind": "entity", "purpose": "Describe a versioned scoring procedure and its harness contract, including optional model judges, with reproducible bindings and explicit validity limits.", "scope_statement": "One identifiable procedure definition across immutable revisions. It owns input and output contracts, scoring recipes, judge qualification requirements and revision comparability. It does not own an evaluation run or authorize operational decisions.", "in_scope": [ "Procedure identity, semantic revision and accountable custodian", "Task applicability, adapters, metrics, rubrics and aggregation recipes", "Optional model judge bindings, calibration and trust policy", "Runtime request and response contracts, replay requirements and qualification evidence", "Definition change, comparability and governed retirement" ], "out_of_scope": [ "WM-AI-003 evaluation run, finding and approval lifecycle", "Dataset curation, training, benchmark ownership and model artifact lifecycle", "Actual scheduling, execution, resource provisioning and external tool effects", "Release approval, certification, regulatory compliance conclusions and universal rankings", "Operational instructions for dangerous evaluations" ], "boundary_notes": [ { "neighbor": "WM-AI-003 AI Model Evaluation", "distinction": "Registry parent only; no incident relation-ledger edges exist at freeze. A proposed REFERENCE binding passes the procedure revision to the evaluation record. Runs, observations, findings and approvals remain external.", "source_refs": [ "SRC-004", "SRC-005" ] }, { "neighbor": "Dataset and model master records", "distinction": "The harness pins immutable material and subject/judge model references. It does not duplicate their ownership or lifecycle. Exact neighbor identifiers require an adopting profile.", "source_refs": [ "SRC-002", "SRC-004" ] }, { "neighbor": "Execution runtime and audit record", "distinction": "This entity describes required behavior and evidence shape. Runtime execution and authoritative event storage are delegated; local inspection is not execution.", "source_refs": [ "SRC-004", "SRC-005" ] }, { "neighbor": "External open specifications", "distinction": "PROV-O and EARL are conceptual mappings. RFC 8785 is an optional serialization binding, not the format-neutral meaning of a score.", "source_refs": [ "SRC-004", "SRC-005", "SRC-006" ] } ] }, "sources": [ { "id": "SRC-001", "title": "AI RMF Playbook - Measure", "organization": "NIST", "url": "https://airc.nist.gov/airmf-resources/playbook/measure/", "version_or_date": "Living guidance accessed 2026-10-21; page announces revision pending", "source_type": "public-authority", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T16:56:03Z", "relevance": "Measure 1.1, 1.2, 2.1 and 2.13 motivate task-specific measurement selection, documentation and reassessment. Local hierarchy and field choices are proposals, not NIST requirements." }, { "id": "SRC-002", "title": "Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile", "organization": "NIST", "url": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf", "version_or_date": "NIST AI 600-1, July 2024", "source_type": "public-authority", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T16:56:03Z", "relevance": "MS-2.5-001, MS-2.5-005, MS-2.11-001 and MS-2.13-001 support limits on generalization, provenance, contamination disclosure and measurement-error assessment. Not a universal scoring standard." }, { "id": "SRC-003", "title": "Building Evaluation Probes into Agentic AI", "organization": "NIST", "url": "https://www.nist.gov/programs-projects/building-evaluation-probes-agentic-ai", "version_or_date": "Created 2026-05-01; updated 2026-05-05 as displayed", "source_type": "first-party-doc", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T16:56:03Z", "relevance": "Overview and Approach describe experimental rubric-based judges, reference corpora and structured verdicts. Supports a judge interface concept, not proven reliability or a required algorithm." }, { "id": "SRC-004", "title": "PROV-O: The PROV Ontology", "organization": "W3C", "url": "https://www.w3.org/TR/prov-o/", "version_or_date": "Recommendation 2013-04-30", "source_type": "ontology", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T16:56:03Z", "relevance": "Plan, Entity, Activity, derivation and qualified association separate a method definition from its use and resulting evidence. Conceptual alignment only." }, { "id": "SRC-005", "title": "Evaluation and Report Language (EARL) 1.0 Schema", "organization": "W3C", "url": "https://www.w3.org/TR/EARL10-Schema/", "version_or_date": "Working Group Note 2017-02-02; work in progress, not Recommendation", "source_type": "schema", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T16:56:03Z", "relevance": "Sections 2 and 3 distinguish assertor, subject, criterion, result and test mode, with explicit indeterminate, inapplicable and untested outcomes. Does not define a complete scoring procedure." }, { "id": "SRC-006", "title": "JSON Canonicalization Scheme (JCS)", "organization": "RFC Editor", "url": "https://www.rfc-editor.org/rfc/rfc8785.html", "version_or_date": "RFC 8785, June 2020; Independent Submission, Informational", "source_type": "standard", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T16:56:03Z", "relevance": "Section 3 supplies an optional JSON byte-canonicalization binding. Canonical bytes do not prove score semantics, source truth or reproducibility." }, { "id": "SRC-007", "title": "Date and Time on the Internet: Timestamps", "organization": "IETF", "url": "https://www.rfc-editor.org/rfc/rfc3339.html", "version_or_date": "RFC 3339, July 2002, section 5.6", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T16:56:03Z", "relevance": "Timestamp representation with seconds and explicit offset. Event, observation and ingestion roles are authored profile distinctions." }, { "id": "SRC-008", "title": "LLM01:2025 Prompt Injection", "organization": "OWASP", "url": "https://genai.owasp.org/llmrisk/llm01-prompt-injection/", "version_or_date": "2025 risk guidance, accessed 2026-10-21", "source_type": "first-party-doc", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T16:56:03Z", "relevance": "Input trust separation and constrained authority motivate policy controls around model judges. No prevention guarantee, exploit recipe or operational attack content is included." } ], "structure": { "bundles": [ { "id": "bundle-definition", "name": "Definition and accountability", "description": "A stable procedure boundary and its intended construct.", "rationale": "Groups related procedure-definition concerns while preserving the external evaluation lifecycle.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-005", "SRC-006" ], "layers": [ { "id": "layer-identity", "name": "Identity and responsibility", "description": "Context for identity and responsibility within a versioned scoring procedure.", "source_refs": [ "SRC-001", "SRC-004", "SRC-005", "SRC-006" ], "findings": [ { "id": "procedure-identity", "name": "Procedure identity and revision", "description": "A harness definition has a persistent identity and immutable semantic revisions. Its configuration is distinct from a particular invocation and from an evaluated model. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-004", "SRC-006" ], "questions": [ { "id": "procedure-identity-q1", "text": "Which master identity and revision identify this scoring procedure?", "kind": "identity", "answer_data": [ "procedure_id", "revision", "master_ref" ] }, { "id": "procedure-identity-q2", "text": "Which scoring mode and supported task family classify this revision?", "kind": "classification", "answer_data": [ "scoring_mode", "task_family" ] }, { "id": "procedure-identity-q3", "text": "Which predecessor and dependency changes produced this revision?", "kind": "provenance", "answer_data": [ "predecessor_ref", "change_record", "dependency_digest" ] } ], "data_elements": [ { "id": "procedure-identity-data-procedure-id", "name": "procedure_id", "description": "Candidate answer field for: Which master identity and revision identify this scoring procedure? Instance constraints must be specified in the adopting profile.", "value_kind": "identifier", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-006" ] }, { "id": "procedure-identity-data-revision", "name": "revision", "description": "Candidate answer field for: Which master identity and revision identify this scoring procedure? Instance constraints must be specified in the adopting profile.", "value_kind": "code", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-006" ] }, { "id": "procedure-identity-data-master-ref", "name": "master_ref", "description": "Candidate answer field for: Which master identity and revision identify this scoring procedure? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-006" ] }, { "id": "procedure-identity-data-scoring-mode", "name": "scoring_mode", "description": "Candidate answer field for: Which scoring mode and supported task family classify this revision? Instance constraints must be specified in the adopting profile.", "value_kind": "code", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-006" ] }, { "id": "procedure-identity-data-task-family", "name": "task_family", "description": "Candidate answer field for: Which scoring mode and supported task family classify this revision? Instance constraints must be specified in the adopting profile.", "value_kind": "code", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-006" ] }, { "id": "procedure-identity-data-predecessor-ref", "name": "predecessor_ref", "description": "Candidate answer field for: Which predecessor and dependency changes produced this revision? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-006" ] }, { "id": "procedure-identity-data-change-record", "name": "change_record", "description": "Candidate answer field for: Which predecessor and dependency changes produced this revision? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-006" ] }, { "id": "procedure-identity-data-dependency-digest", "name": "dependency_digest", "description": "Candidate answer field for: Which predecessor and dependency changes produced this revision? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-006" ] } ], "artifacts": [ { "id": "procedure-identity-artifact", "name": "Procedure revision manifest", "description": "Versioned definition or evidence-index artifact for procedure identity and revision. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-004", "SRC-006" ] } ], "inline_only_rationale": null }, { "id": "ownership-boundary", "name": "Accountability and evaluation binding", "description": "The procedure custodian governs definitions; the evaluation owner governs runs, findings and approvals. A registry parent is contextual and does not establish executable inheritance. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ], "questions": [ { "id": "ownership-boundary-q1", "text": "Which accountable role maintains the procedure and which role owns evaluation decisions?", "kind": "ownership", "answer_data": [ "custodian_ref", "evaluation_owner_ref" ] }, { "id": "ownership-boundary-q2", "text": "Which evaluation record may reference this revision without transferring its lifecycle here?", "kind": "relationship", "answer_data": [ "evaluation_model_ref", "binding_profile" ] }, { "id": "ownership-boundary-q3", "text": "What evidence permits a custodian to issue a new procedure revision?", "kind": "authority", "answer_data": [ "delegation_ref", "review_record", "permission_scope" ] } ], "data_elements": [ { "id": "ownership-boundary-data-custodian-ref", "name": "custodian_ref", "description": "Candidate answer field for: Which accountable role maintains the procedure and which role owns evaluation decisions? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ] }, { "id": "ownership-boundary-data-evaluation-owner-ref", "name": "evaluation_owner_ref", "description": "Candidate answer field for: Which accountable role maintains the procedure and which role owns evaluation decisions? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ] }, { "id": "ownership-boundary-data-evaluation-model-ref", "name": "evaluation_model_ref", "description": "Candidate answer field for: Which evaluation record may reference this revision without transferring its lifecycle here? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ] }, { "id": "ownership-boundary-data-binding-profile", "name": "binding_profile", "description": "Candidate answer field for: Which evaluation record may reference this revision without transferring its lifecycle here? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ] }, { "id": "ownership-boundary-data-delegation-ref", "name": "delegation_ref", "description": "Candidate answer field for: What evidence permits a custodian to issue a new procedure revision? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ] }, { "id": "ownership-boundary-data-review-record", "name": "review_record", "description": "Candidate answer field for: What evidence permits a custodian to issue a new procedure revision? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ] }, { "id": "ownership-boundary-data-permission-scope", "name": "permission_scope", "description": "Candidate answer field for: What evidence permits a custodian to issue a new procedure revision? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ] } ], "artifacts": [ { "id": "ownership-boundary-artifact", "name": "Responsibility and binding record", "description": "Versioned definition or evidence-index artifact for accountability and evaluation binding. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-001", "SRC-004", "SRC-005" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-applicability", "name": "Measurement applicability", "description": "Context for measurement applicability within a versioned scoring procedure.", "source_refs": [ "SRC-001", "SRC-002" ], "findings": [ { "id": "intended-construct", "name": "Intended construct and applicability", "description": "The author declares what the procedure intends to measure and where it is unsupported. A computed value alone cannot establish suitability for an evaluation decision. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-001", "SRC-002" ], "questions": [ { "id": "intended-construct-q1", "text": "What construct is measured and what observable evidence is used as its proxy?", "kind": "definition", "answer_data": [ "construct", "proxy_definition", "validity_evidence_ref" ] }, { "id": "intended-construct-q2", "text": "Which task, language, modality and deployment contexts are inside the claimed scope?", "kind": "constraint", "answer_data": [ "applicability_profile", "excluded_contexts" ] }, { "id": "intended-construct-q3", "text": "Which unmet conditions require rejection or an explicitly unsupported result?", "kind": "exception", "answer_data": [ "precondition_rules", "unsupported_reason_codes" ] } ], "data_elements": [ { "id": "intended-construct-data-construct", "name": "construct", "description": "Candidate answer field for: What construct is measured and what observable evidence is used as its proxy? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002" ] }, { "id": "intended-construct-data-proxy-definition", "name": "proxy_definition", "description": "Candidate answer field for: What construct is measured and what observable evidence is used as its proxy? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002" ] }, { "id": "intended-construct-data-validity-evidence-ref", "name": "validity_evidence_ref", "description": "Candidate answer field for: What construct is measured and what observable evidence is used as its proxy? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002" ] }, { "id": "intended-construct-data-applicability-profile", "name": "applicability_profile", "description": "Candidate answer field for: Which task, language, modality and deployment contexts are inside the claimed scope? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002" ] }, { "id": "intended-construct-data-excluded-contexts", "name": "excluded_contexts", "description": "Candidate answer field for: Which task, language, modality and deployment contexts are inside the claimed scope? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002" ] }, { "id": "intended-construct-data-precondition-rules", "name": "precondition_rules", "description": "Candidate answer field for: Which unmet conditions require rejection or an explicitly unsupported result? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002" ] }, { "id": "intended-construct-data-unsupported-reason-codes", "name": "unsupported_reason_codes", "description": "Candidate answer field for: Which unmet conditions require rejection or an explicitly unsupported result? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002" ] } ], "artifacts": [ { "id": "intended-construct-artifact", "name": "Applicability statement", "description": "Versioned definition or evidence-index artifact for intended construct and applicability. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-001", "SRC-002" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-inputs", "name": "Inputs and preparation", "description": "Bindings and transformations that precede scoring.", "rationale": "Groups related procedure-definition concerns while preserving the external evaluation lifecycle.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-005", "SRC-008" ], "layers": [ { "id": "layer-material", "name": "Input roles and material bindings", "description": "Context for input roles and material bindings within a versioned scoring procedure.", "source_refs": [ "SRC-002", "SRC-004", "SRC-005" ], "findings": [ { "id": "input-contract", "name": "Input roles and admissibility", "description": "The adapter declares subject output, reference material, item identity and task context separately. Missing labels, reordered items and unavailable material cannot silently become valid inputs. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-005", "SRC-004" ], "questions": [ { "id": "input-contract-q1", "text": "Which input roles and schema versions are required for this scoring mode?", "kind": "composition", "answer_data": [ "input_roles", "input_schema_ref", "conditional_requirements" ] }, { "id": "input-contract-q2", "text": "How are item identity, output alignment and missing required inputs checked?", "kind": "validation", "answer_data": [ "join_key_rule", "alignment_checks", "rejection_codes" ] }, { "id": "input-contract-q3", "text": "Which adapter transformations are permitted and which information losses must be reported?", "kind": "interoperability", "answer_data": [ "adapter_revision", "transformation_rules", "loss_report_schema" ] } ], "data_elements": [ { "id": "input-contract-data-input-roles", "name": "input_roles", "description": "Candidate answer field for: Which input roles and schema versions are required for this scoring mode? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] }, { "id": "input-contract-data-input-schema-ref", "name": "input_schema_ref", "description": "Candidate answer field for: Which input roles and schema versions are required for this scoring mode? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] }, { "id": "input-contract-data-conditional-requirements", "name": "conditional_requirements", "description": "Candidate answer field for: Which input roles and schema versions are required for this scoring mode? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] }, { "id": "input-contract-data-join-key-rule", "name": "join_key_rule", "description": "Candidate answer field for: How are item identity, output alignment and missing required inputs checked? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] }, { "id": "input-contract-data-alignment-checks", "name": "alignment_checks", "description": "Candidate answer field for: How are item identity, output alignment and missing required inputs checked? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] }, { "id": "input-contract-data-rejection-codes", "name": "rejection_codes", "description": "Candidate answer field for: How are item identity, output alignment and missing required inputs checked? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] }, { "id": "input-contract-data-adapter-revision", "name": "adapter_revision", "description": "Candidate answer field for: Which adapter transformations are permitted and which information losses must be reported? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] }, { "id": "input-contract-data-transformation-rules", "name": "transformation_rules", "description": "Candidate answer field for: Which adapter transformations are permitted and which information losses must be reported? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] }, { "id": "input-contract-data-loss-report-schema", "name": "loss_report_schema", "description": "Candidate answer field for: Which adapter transformations are permitted and which information losses must be reported? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004" ] } ], "artifacts": [ { "id": "input-contract-artifact", "name": "Input and adapter contract", "description": "Versioned definition or evidence-index artifact for input roles and admissibility. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-005", "SRC-004" ] } ], "inline_only_rationale": null }, { "id": "test-material-binding", "name": "Test material and separation rules", "description": "The definition references externally mastered test material and declares split eligibility, exposure restrictions and contamination checks. It neither curates the dataset nor certifies absence of contamination. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-002", "SRC-004" ], "questions": [ { "id": "test-material-binding-q1", "text": "Which immutable dataset or reference-corpus revision and selection rule does the procedure accept?", "kind": "provenance", "answer_data": [ "material_ref", "material_revision", "selection_rule" ] }, { "id": "test-material-binding-q2", "text": "What training, tuning or judge exposure would undermine the proposed measurement?", "kind": "quality", "answer_data": [ "exposure_policy", "contamination_evidence_ref", "unknown_exposure" ] }, { "id": "test-material-binding-q3", "text": "Which actor may view held-out labels or reference answers during each scoring phase?", "kind": "access", "answer_data": [ "label_access_matrix", "blinding_rule", "access_policy_ref" ] } ], "data_elements": [ { "id": "test-material-binding-data-material-ref", "name": "material_ref", "description": "Candidate answer field for: Which immutable dataset or reference-corpus revision and selection rule does the procedure accept? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] }, { "id": "test-material-binding-data-material-revision", "name": "material_revision", "description": "Candidate answer field for: Which immutable dataset or reference-corpus revision and selection rule does the procedure accept? Instance constraints must be specified in the adopting profile.", "value_kind": "code", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] }, { "id": "test-material-binding-data-selection-rule", "name": "selection_rule", "description": "Candidate answer field for: Which immutable dataset or reference-corpus revision and selection rule does the procedure accept? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] }, { "id": "test-material-binding-data-exposure-policy", "name": "exposure_policy", "description": "Candidate answer field for: What training, tuning or judge exposure would undermine the proposed measurement? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] }, { "id": "test-material-binding-data-contamination-evidence-ref", "name": "contamination_evidence_ref", "description": "Candidate answer field for: What training, tuning or judge exposure would undermine the proposed measurement? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] }, { "id": "test-material-binding-data-unknown-exposure", "name": "unknown_exposure", "description": "Candidate answer field for: What training, tuning or judge exposure would undermine the proposed measurement? Instance constraints must be specified in the adopting profile.", "value_kind": "boolean", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] }, { "id": "test-material-binding-data-label-access-matrix", "name": "label_access_matrix", "description": "Candidate answer field for: Which actor may view held-out labels or reference answers during each scoring phase? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] }, { "id": "test-material-binding-data-blinding-rule", "name": "blinding_rule", "description": "Candidate answer field for: Which actor may view held-out labels or reference answers during each scoring phase? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] }, { "id": "test-material-binding-data-access-policy-ref", "name": "access_policy_ref", "description": "Candidate answer field for: Which actor may view held-out labels or reference answers during each scoring phase? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004" ] } ], "artifacts": [ { "id": "test-material-binding-artifact", "name": "Test material binding profile", "description": "Versioned definition or evidence-index artifact for test material and separation rules. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-002", "SRC-004" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-preparation", "name": "Preparation constraints", "description": "Context for preparation constraints within a versioned scoring procedure.", "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ], "findings": [ { "id": "preparation-policy", "name": "Preprocessing and resource policy", "description": "A proposed preparation contract pins parsing, normalization, truncation and resource ceilings before scoring. These settings can affect meaning and belong in comparison criteria. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ], "questions": [ { "id": "preparation-policy-q1", "text": "Which ordered normalization and extraction steps preserve or change score-relevant meaning?", "kind": "process", "answer_data": [ "preprocessing_steps", "parser_revision", "semantic_loss_rules" ] }, { "id": "preparation-policy-q2", "text": "Which input limits and truncation behavior apply to every supported modality?", "kind": "requirement", "answer_data": [ "input_limits", "truncation_policy", "unsupported_modality_rule" ] }, { "id": "preparation-policy-q3", "text": "Which isolation and resource restrictions must a runtime attest before using this definition?", "kind": "security", "answer_data": [ "runtime_policy_ref", "resource_limits", "isolation_evidence_ref" ] } ], "data_elements": [ { "id": "preparation-policy-data-preprocessing-steps", "name": "preprocessing_steps", "description": "Candidate answer field for: Which ordered normalization and extraction steps preserve or change score-relevant meaning? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] }, { "id": "preparation-policy-data-parser-revision", "name": "parser_revision", "description": "Candidate answer field for: Which ordered normalization and extraction steps preserve or change score-relevant meaning? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] }, { "id": "preparation-policy-data-semantic-loss-rules", "name": "semantic_loss_rules", "description": "Candidate answer field for: Which ordered normalization and extraction steps preserve or change score-relevant meaning? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] }, { "id": "preparation-policy-data-input-limits", "name": "input_limits", "description": "Candidate answer field for: Which input limits and truncation behavior apply to every supported modality? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] }, { "id": "preparation-policy-data-truncation-policy", "name": "truncation_policy", "description": "Candidate answer field for: Which input limits and truncation behavior apply to every supported modality? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] }, { "id": "preparation-policy-data-unsupported-modality-rule", "name": "unsupported_modality_rule", "description": "Candidate answer field for: Which input limits and truncation behavior apply to every supported modality? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] }, { "id": "preparation-policy-data-runtime-policy-ref", "name": "runtime_policy_ref", "description": "Candidate answer field for: Which isolation and resource restrictions must a runtime attest before using this definition? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] }, { "id": "preparation-policy-data-resource-limits", "name": "resource_limits", "description": "Candidate answer field for: Which isolation and resource restrictions must a runtime attest before using this definition? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] }, { "id": "preparation-policy-data-isolation-evidence-ref", "name": "isolation_evidence_ref", "description": "Candidate answer field for: Which isolation and resource restrictions must a runtime attest before using this definition? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] } ], "artifacts": [ { "id": "preparation-policy-artifact", "name": "Preparation and runtime constraint profile", "description": "Versioned definition or evidence-index artifact for preprocessing and resource policy. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-001", "SRC-002", "SRC-008" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-scoring", "name": "Scoring semantics", "description": "Formula, rubric and aggregation definitions.", "rationale": "Groups related procedure-definition concerns while preserving the external evaluation lifecycle.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005" ], "layers": [ { "id": "layer-criteria", "name": "Criteria and rubrics", "description": "Context for criteria and rubrics within a versioned scoring procedure.", "source_refs": [ "SRC-001", "SRC-003", "SRC-005" ], "findings": [ { "id": "metric-contract", "name": "Metric formula and score semantics", "description": "Each selected metric needs a precise value domain, orientation and interpretation. Metric definitions can be externally referenced; this procedure owns their binding, parameterization and composition. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-001", "SRC-005" ], "questions": [ { "id": "metric-contract-q1", "text": "Which formula, value type, unit and direction define each returned score?", "kind": "measurement", "answer_data": [ "metric_definition_ref", "value_domain", "unit", "orientation" ] }, { "id": "metric-contract-q2", "text": "How are ties, boundary values and undefined arithmetic handled?", "kind": "constraint", "answer_data": [ "tie_rule", "boundary_rule", "undefined_rule" ] }, { "id": "metric-contract-q3", "text": "Which thresholds yield a local criterion outcome and which release decisions remain external?", "kind": "decision", "answer_data": [ "criterion_thresholds", "decision_policy_ref", "non_authorized_decisions" ] } ], "data_elements": [ { "id": "metric-contract-data-metric-definition-ref", "name": "metric_definition_ref", "description": "Candidate answer field for: Which formula, value type, unit and direction define each returned score? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-value-domain", "name": "value_domain", "description": "Candidate answer field for: Which formula, value type, unit and direction define each returned score? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-unit", "name": "unit", "description": "Candidate answer field for: Which formula, value type, unit and direction define each returned score? Instance constraints must be specified in the adopting profile.", "value_kind": "code", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-orientation", "name": "orientation", "description": "Candidate answer field for: Which formula, value type, unit and direction define each returned score? Instance constraints must be specified in the adopting profile.", "value_kind": "code", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-tie-rule", "name": "tie_rule", "description": "Candidate answer field for: How are ties, boundary values and undefined arithmetic handled? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-boundary-rule", "name": "boundary_rule", "description": "Candidate answer field for: How are ties, boundary values and undefined arithmetic handled? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-undefined-rule", "name": "undefined_rule", "description": "Candidate answer field for: How are ties, boundary values and undefined arithmetic handled? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-criterion-thresholds", "name": "criterion_thresholds", "description": "Candidate answer field for: Which thresholds yield a local criterion outcome and which release decisions remain external? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-decision-policy-ref", "name": "decision_policy_ref", "description": "Candidate answer field for: Which thresholds yield a local criterion outcome and which release decisions remain external? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "metric-contract-data-non-authorized-decisions", "name": "non_authorized_decisions", "description": "Candidate answer field for: Which thresholds yield a local criterion outcome and which release decisions remain external? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-005" ] } ], "artifacts": [ { "id": "metric-contract-artifact", "name": "Metric binding and interpretation sheet", "description": "Versioned definition or evidence-index artifact for metric formula and score semantics. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-001", "SRC-005" ] } ], "inline_only_rationale": null }, { "id": "rubric-contract", "name": "Rubric and reference judgement", "description": "A judge rubric describes criteria, scale anchors and admissible evidence. Exemplars and reference judgements are versioned inputs, not unquestionable ground truth. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-003", "SRC-005" ], "questions": [ { "id": "rubric-contract-q1", "text": "Which criterion wording and scale anchors constrain the judgement?", "kind": "definition", "answer_data": [ "rubric_revision", "criteria", "scale_anchors" ] }, { "id": "rubric-contract-q2", "text": "Which reference examples and counterexamples establish the intended interpretation?", "kind": "evidence", "answer_data": [ "exemplar_refs", "reference_provenance", "disputed_labels" ] }, { "id": "rubric-contract-q3", "text": "When should the rubric require abstention or referral instead of a numeric score?", "kind": "exception", "answer_data": [ "abstention_conditions", "referral_policy_ref" ] } ], "data_elements": [ { "id": "rubric-contract-data-rubric-revision", "name": "rubric_revision", "description": "Candidate answer field for: Which criterion wording and scale anchors constrain the judgement? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "rubric-contract-data-criteria", "name": "criteria", "description": "Candidate answer field for: Which criterion wording and scale anchors constrain the judgement? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "rubric-contract-data-scale-anchors", "name": "scale_anchors", "description": "Candidate answer field for: Which criterion wording and scale anchors constrain the judgement? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "rubric-contract-data-exemplar-refs", "name": "exemplar_refs", "description": "Candidate answer field for: Which reference examples and counterexamples establish the intended interpretation? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "rubric-contract-data-reference-provenance", "name": "reference_provenance", "description": "Candidate answer field for: Which reference examples and counterexamples establish the intended interpretation? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "rubric-contract-data-disputed-labels", "name": "disputed_labels", "description": "Candidate answer field for: Which reference examples and counterexamples establish the intended interpretation? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "rubric-contract-data-abstention-conditions", "name": "abstention_conditions", "description": "Candidate answer field for: When should the rubric require abstention or referral instead of a numeric score? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "rubric-contract-data-referral-policy-ref", "name": "referral_policy_ref", "description": "Candidate answer field for: When should the rubric require abstention or referral instead of a numeric score? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] } ], "artifacts": [ { "id": "rubric-contract-artifact", "name": "Rubric and anchor specification", "description": "Versioned definition or evidence-index artifact for rubric and reference judgement. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-003", "SRC-005" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-aggregation", "name": "Aggregation policy", "description": "Context for aggregation policy within a versioned scoring procedure.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ], "findings": [ { "id": "aggregate-contract", "name": "Aggregation and denominator policy", "description": "The proposed aggregation recipe declares weighting, eligible populations and missingness before results are inspected. Summary values preserve denominator and component context. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ], "questions": [ { "id": "aggregate-contract-q1", "text": "How are item values, weights and group summaries combined?", "kind": "measurement", "answer_data": [ "aggregation_formula", "weight_rule", "grouping_rule" ] }, { "id": "aggregate-contract-q2", "text": "Which attempted, eligible, valid and excluded counts accompany each summary?", "kind": "quality", "answer_data": [ "denominator_schema", "exclusion_policy", "minimum_support_rule" ] }, { "id": "aggregate-contract-q3", "text": "How are repeated items, correlated samples and undefined group summaries treated?", "kind": "validation", "answer_data": [ "duplicate_rule", "dependence_assumption", "undefined_group_rule" ] } ], "data_elements": [ { "id": "aggregate-contract-data-aggregation-formula", "name": "aggregation_formula", "description": "Candidate answer field for: How are item values, weights and group summaries combined? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "aggregate-contract-data-weight-rule", "name": "weight_rule", "description": "Candidate answer field for: How are item values, weights and group summaries combined? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "aggregate-contract-data-grouping-rule", "name": "grouping_rule", "description": "Candidate answer field for: How are item values, weights and group summaries combined? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "aggregate-contract-data-denominator-schema", "name": "denominator_schema", "description": "Candidate answer field for: Which attempted, eligible, valid and excluded counts accompany each summary? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "aggregate-contract-data-exclusion-policy", "name": "exclusion_policy", "description": "Candidate answer field for: Which attempted, eligible, valid and excluded counts accompany each summary? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "aggregate-contract-data-minimum-support-rule", "name": "minimum_support_rule", "description": "Candidate answer field for: Which attempted, eligible, valid and excluded counts accompany each summary? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "aggregate-contract-data-duplicate-rule", "name": "duplicate_rule", "description": "Candidate answer field for: How are repeated items, correlated samples and undefined group summaries treated? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "aggregate-contract-data-dependence-assumption", "name": "dependence_assumption", "description": "Candidate answer field for: How are repeated items, correlated samples and undefined group summaries treated? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "aggregate-contract-data-undefined-group-rule", "name": "undefined_group_rule", "description": "Candidate answer field for: How are repeated items, correlated samples and undefined group summaries treated? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] } ], "artifacts": [ { "id": "aggregate-contract-artifact", "name": "Aggregation recipe", "description": "Versioned definition or evidence-index artifact for aggregation and denominator policy. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-judges", "name": "Model judge qualification", "description": "Optional model judges and their limitations.", "rationale": "Groups related procedure-definition concerns while preserving the external evaluation lifecycle.", "source_refs": [ "SRC-002", "SRC-003", "SRC-004", "SRC-005", "SRC-008" ], "layers": [ { "id": "layer-configuration", "name": "Configuration and calibration", "description": "Context for configuration and calibration within a versioned scoring procedure.", "source_refs": [ "SRC-002", "SRC-003", "SRC-004" ], "findings": [ { "id": "judge-binding", "name": "Model judge configuration", "description": "A model judge is an optional scoring dependency with pinned model reference, prompt, rubric and output parser. Unknown remote revision or stochastic settings restrict replay claims. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-003", "SRC-004" ], "questions": [ { "id": "judge-binding-q1", "text": "Which judge model, prompt, rubric and parser revisions form the scoring dependency?", "kind": "composition", "answer_data": [ "judge_model_ref", "prompt_revision", "rubric_ref", "parser_ref" ] }, { "id": "judge-binding-q2", "text": "Which decoding settings, seeds and availability assumptions apply to the judge?", "kind": "state", "answer_data": [ "sampling_configuration", "seed_policy", "availability_contract" ] }, { "id": "judge-binding-q3", "text": "How is a resolved judge revision captured when an endpoint alias can change?", "kind": "provenance", "answer_data": [ "resolution_rule", "resolved_revision_field", "unresolved_revision_policy" ] } ], "data_elements": [ { "id": "judge-binding-data-judge-model-ref", "name": "judge_model_ref", "description": "Candidate answer field for: Which judge model, prompt, rubric and parser revisions form the scoring dependency? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-prompt-revision", "name": "prompt_revision", "description": "Candidate answer field for: Which judge model, prompt, rubric and parser revisions form the scoring dependency? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-rubric-ref", "name": "rubric_ref", "description": "Candidate answer field for: Which judge model, prompt, rubric and parser revisions form the scoring dependency? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-parser-ref", "name": "parser_ref", "description": "Candidate answer field for: Which judge model, prompt, rubric and parser revisions form the scoring dependency? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-sampling-configuration", "name": "sampling_configuration", "description": "Candidate answer field for: Which decoding settings, seeds and availability assumptions apply to the judge? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-seed-policy", "name": "seed_policy", "description": "Candidate answer field for: Which decoding settings, seeds and availability assumptions apply to the judge? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-availability-contract", "name": "availability_contract", "description": "Candidate answer field for: Which decoding settings, seeds and availability assumptions apply to the judge? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-resolution-rule", "name": "resolution_rule", "description": "Candidate answer field for: How is a resolved judge revision captured when an endpoint alias can change? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-resolved-revision-field", "name": "resolved_revision_field", "description": "Candidate answer field for: How is a resolved judge revision captured when an endpoint alias can change? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] }, { "id": "judge-binding-data-unresolved-revision-policy", "name": "unresolved_revision_policy", "description": "Candidate answer field for: How is a resolved judge revision captured when an endpoint alias can change? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-004" ] } ], "artifacts": [ { "id": "judge-binding-artifact", "name": "Judge configuration manifest", "description": "Versioned definition or evidence-index artifact for model judge configuration. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-003", "SRC-004" ] } ], "inline_only_rationale": null }, { "id": "judge-calibration", "name": "Judge calibration and bias checks", "description": "A proposed judge qualification profile records reference-review agreement and sensitivity tests before adoption. It makes no universal accuracy claim and sets no default acceptance percentage. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-002", "SRC-003" ], "questions": [ { "id": "judge-calibration-q1", "text": "Which independent reference judgements and acceptance criteria qualify this judge for this task?", "kind": "validation", "answer_data": [ "reference_review_ref", "qualification_metrics", "acceptance_rule" ] }, { "id": "judge-calibration-q2", "text": "Which order, verbosity, language and self-preference sensitivity checks are required?", "kind": "quality", "answer_data": [ "sensitivity_plan", "subgroup_plan", "unsupported_slices" ] }, { "id": "judge-calibration-q3", "text": "Which rubric, judge or context changes invalidate prior qualification?", "kind": "lifecycle", "answer_data": [ "requalification_triggers", "calibration_validity_rule", "review_owner_ref" ] } ], "data_elements": [ { "id": "judge-calibration-data-reference-review-ref", "name": "reference_review_ref", "description": "Candidate answer field for: Which independent reference judgements and acceptance criteria qualify this judge for this task? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] }, { "id": "judge-calibration-data-qualification-metrics", "name": "qualification_metrics", "description": "Candidate answer field for: Which independent reference judgements and acceptance criteria qualify this judge for this task? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] }, { "id": "judge-calibration-data-acceptance-rule", "name": "acceptance_rule", "description": "Candidate answer field for: Which independent reference judgements and acceptance criteria qualify this judge for this task? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] }, { "id": "judge-calibration-data-sensitivity-plan", "name": "sensitivity_plan", "description": "Candidate answer field for: Which order, verbosity, language and self-preference sensitivity checks are required? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] }, { "id": "judge-calibration-data-subgroup-plan", "name": "subgroup_plan", "description": "Candidate answer field for: Which order, verbosity, language and self-preference sensitivity checks are required? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] }, { "id": "judge-calibration-data-unsupported-slices", "name": "unsupported_slices", "description": "Candidate answer field for: Which order, verbosity, language and self-preference sensitivity checks are required? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] }, { "id": "judge-calibration-data-requalification-triggers", "name": "requalification_triggers", "description": "Candidate answer field for: Which rubric, judge or context changes invalidate prior qualification? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] }, { "id": "judge-calibration-data-calibration-validity-rule", "name": "calibration_validity_rule", "description": "Candidate answer field for: Which rubric, judge or context changes invalidate prior qualification? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] }, { "id": "judge-calibration-data-review-owner-ref", "name": "review_owner_ref", "description": "Candidate answer field for: Which rubric, judge or context changes invalidate prior qualification? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] } ], "artifacts": [ { "id": "judge-calibration-artifact", "name": "Judge qualification plan and evidence index", "description": "Versioned definition or evidence-index artifact for judge calibration and bias checks. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-002", "SRC-003" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-trust", "name": "Trust and review controls", "description": "Context for trust and review controls within a versioned scoring procedure.", "source_refs": [ "SRC-003", "SRC-005", "SRC-008" ], "findings": [ { "id": "judge-trust", "name": "Judge trust boundary and review route", "description": "Scored content is untrusted data. The harness declares least-authority requirements, evidence handling and escalation policy; dangerous evaluations remain at policy level. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ], "questions": [ { "id": "judge-trust-q1", "text": "How must runtime policy isolate evaluated content from judge instructions and privileged actions?", "kind": "security", "answer_data": [ "trust_boundary_policy", "allowed_capabilities", "output_validation_rule" ] }, { "id": "judge-trust-q2", "text": "Which sensitive input or rationale content may be withheld, minimized or redacted?", "kind": "privacy", "answer_data": [ "data_handling_policy_ref", "redaction_rule", "authorized_viewers" ] }, { "id": "judge-trust-q3", "text": "Who may review disputed judge outputs without rewriting the original verdict?", "kind": "authority", "answer_data": [ "review_route_ref", "reviewer_role_ref", "supersession_rule" ] } ], "data_elements": [ { "id": "judge-trust-data-trust-boundary-policy", "name": "trust_boundary_policy", "description": "Candidate answer field for: How must runtime policy isolate evaluated content from judge instructions and privileged actions? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] }, { "id": "judge-trust-data-allowed-capabilities", "name": "allowed_capabilities", "description": "Candidate answer field for: How must runtime policy isolate evaluated content from judge instructions and privileged actions? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] }, { "id": "judge-trust-data-output-validation-rule", "name": "output_validation_rule", "description": "Candidate answer field for: How must runtime policy isolate evaluated content from judge instructions and privileged actions? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] }, { "id": "judge-trust-data-data-handling-policy-ref", "name": "data_handling_policy_ref", "description": "Candidate answer field for: Which sensitive input or rationale content may be withheld, minimized or redacted? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] }, { "id": "judge-trust-data-redaction-rule", "name": "redaction_rule", "description": "Candidate answer field for: Which sensitive input or rationale content may be withheld, minimized or redacted? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] }, { "id": "judge-trust-data-authorized-viewers", "name": "authorized_viewers", "description": "Candidate answer field for: Which sensitive input or rationale content may be withheld, minimized or redacted? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] }, { "id": "judge-trust-data-review-route-ref", "name": "review_route_ref", "description": "Candidate answer field for: Who may review disputed judge outputs without rewriting the original verdict? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] }, { "id": "judge-trust-data-reviewer-role-ref", "name": "reviewer_role_ref", "description": "Candidate answer field for: Who may review disputed judge outputs without rewriting the original verdict? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] }, { "id": "judge-trust-data-supersession-rule", "name": "supersession_rule", "description": "Candidate answer field for: Who may review disputed judge outputs without rewriting the original verdict? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] } ], "artifacts": [ { "id": "judge-trust-artifact", "name": "Judge security and review policy", "description": "Versioned definition or evidence-index artifact for judge trust boundary and review route. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-008", "SRC-003", "SRC-005" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-invocation", "name": "Invocation and evidence contract", "description": "Required runtime behavior and evidence shapes without run ownership.", "rationale": "Groups related procedure-definition concerns while preserving the external evaluation lifecycle.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008" ], "layers": [ { "id": "layer-runtime", "name": "Invocation and response", "description": "Context for invocation and response within a versioned scoring procedure.", "source_refs": [ "SRC-004", "SRC-005", "SRC-007", "SRC-008" ], "findings": [ { "id": "execution-contract", "name": "Invocation recipe and failure behavior", "description": "The procedure defines the request contract and expected failure semantics. Execution, scheduling, retries actually taken and run state are owned by the evaluation runtime. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ], "questions": [ { "id": "execution-contract-q1", "text": "Which steps and dependency checks must precede a scoring invocation?", "kind": "process", "answer_data": [ "invocation_steps", "dependency_checks", "runtime_binding_ref" ] }, { "id": "execution-contract-q2", "text": "How must timeouts, malformed responses, refusals and cancellation be represented?", "kind": "exception", "answer_data": [ "failure_taxonomy", "failure_mapping", "cancellation_rule" ] }, { "id": "execution-contract-q3", "text": "Which timeout, retry and cache rules preserve the declared observation unit?", "kind": "temporal", "answer_data": [ "timeout_rule", "retry_policy", "cache_key_contract" ] } ], "data_elements": [ { "id": "execution-contract-data-invocation-steps", "name": "invocation_steps", "description": "Candidate answer field for: Which steps and dependency checks must precede a scoring invocation? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "execution-contract-data-dependency-checks", "name": "dependency_checks", "description": "Candidate answer field for: Which steps and dependency checks must precede a scoring invocation? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "execution-contract-data-runtime-binding-ref", "name": "runtime_binding_ref", "description": "Candidate answer field for: Which steps and dependency checks must precede a scoring invocation? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "execution-contract-data-failure-taxonomy", "name": "failure_taxonomy", "description": "Candidate answer field for: How must timeouts, malformed responses, refusals and cancellation be represented? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "execution-contract-data-failure-mapping", "name": "failure_mapping", "description": "Candidate answer field for: How must timeouts, malformed responses, refusals and cancellation be represented? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "execution-contract-data-cancellation-rule", "name": "cancellation_rule", "description": "Candidate answer field for: How must timeouts, malformed responses, refusals and cancellation be represented? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "execution-contract-data-timeout-rule", "name": "timeout_rule", "description": "Candidate answer field for: Which timeout, retry and cache rules preserve the declared observation unit? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "execution-contract-data-retry-policy", "name": "retry_policy", "description": "Candidate answer field for: Which timeout, retry and cache rules preserve the declared observation unit? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "execution-contract-data-cache-key-contract", "name": "cache_key_contract", "description": "Candidate answer field for: Which timeout, retry and cache rules preserve the declared observation unit? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] } ], "artifacts": [ { "id": "execution-contract-artifact", "name": "Invocation contract", "description": "Versioned definition or evidence-index artifact for invocation recipe and failure behavior. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] } ], "inline_only_rationale": null }, { "id": "result-envelope", "name": "Result envelope and provenance linkage", "description": "A response schema carries item and procedure references, outcome state, optional score and evidence pointers. The actual result is mastered with its evaluation run, not as the harness definition. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ], "questions": [ { "id": "result-envelope-q1", "text": "Which invocation, item and procedure revision identifiers must each response retain?", "kind": "identity", "answer_data": [ "response_identity_schema", "required_reference_roles" ] }, { "id": "result-envelope-q2", "text": "How are valid score, failure, indeterminate, inapplicable and untested outcomes distinguished?", "kind": "state", "answer_data": [ "outcome_schema_ref", "score_presence_rule", "failure_detail_rule" ] }, { "id": "result-envelope-q3", "text": "Which event and observation times accompany provenance without serving as identity?", "kind": "temporal", "answer_data": [ "event_time_field", "observation_time_field", "timestamp_profile" ] } ], "data_elements": [ { "id": "result-envelope-data-response-identity-schema", "name": "response_identity_schema", "description": "Candidate answer field for: Which invocation, item and procedure revision identifiers must each response retain? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] }, { "id": "result-envelope-data-required-reference-roles", "name": "required_reference_roles", "description": "Candidate answer field for: Which invocation, item and procedure revision identifiers must each response retain? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] }, { "id": "result-envelope-data-outcome-schema-ref", "name": "outcome_schema_ref", "description": "Candidate answer field for: How are valid score, failure, indeterminate, inapplicable and untested outcomes distinguished? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] }, { "id": "result-envelope-data-score-presence-rule", "name": "score_presence_rule", "description": "Candidate answer field for: How are valid score, failure, indeterminate, inapplicable and untested outcomes distinguished? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] }, { "id": "result-envelope-data-failure-detail-rule", "name": "failure_detail_rule", "description": "Candidate answer field for: How are valid score, failure, indeterminate, inapplicable and untested outcomes distinguished? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] }, { "id": "result-envelope-data-event-time-field", "name": "event_time_field", "description": "Candidate answer field for: Which event and observation times accompany provenance without serving as identity? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] }, { "id": "result-envelope-data-observation-time-field", "name": "observation_time_field", "description": "Candidate answer field for: Which event and observation times accompany provenance without serving as identity? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] }, { "id": "result-envelope-data-timestamp-profile", "name": "timestamp_profile", "description": "Candidate answer field for: Which event and observation times accompany provenance without serving as identity? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] } ], "artifacts": [ { "id": "result-envelope-artifact", "name": "Result envelope schema proposal", "description": "Versioned definition or evidence-index artifact for result envelope and provenance linkage. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-005", "SRC-004", "SRC-007" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-reproducibility", "name": "Replay requirements", "description": "Context for replay requirements within a versioned scoring procedure.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ], "findings": [ { "id": "replay-contract", "name": "Replay and uncertainty contract", "description": "A proposed reproducibility profile distinguishes exact replay from distributional agreement. Environment dependencies and uncertainty methods are declared rather than inferred from a seed. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ], "questions": [ { "id": "replay-contract-q1", "text": "Which code, dependencies, material, environment and configuration pins are needed for replay?", "kind": "requirement", "answer_data": [ "replay_manifest_schema", "required_pins", "unavailable_dependencies" ] }, { "id": "replay-contract-q2", "text": "Which variability sources and uncertainty method apply to this score and sampling unit?", "kind": "measurement", "answer_data": [ "uncertainty_method_ref", "sampling_unit", "variability_sources", "interval_semantics" ] }, { "id": "replay-contract-q3", "text": "What evidence and tolerances distinguish successful replay from a merely similar result?", "kind": "validation", "answer_data": [ "replay_acceptance_rule", "tolerance_profile", "reproducibility_limitations" ] } ], "data_elements": [ { "id": "replay-contract-data-replay-manifest-schema", "name": "replay_manifest_schema", "description": "Candidate answer field for: Which code, dependencies, material, environment and configuration pins are needed for replay? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-required-pins", "name": "required_pins", "description": "Candidate answer field for: Which code, dependencies, material, environment and configuration pins are needed for replay? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-unavailable-dependencies", "name": "unavailable_dependencies", "description": "Candidate answer field for: Which code, dependencies, material, environment and configuration pins are needed for replay? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-uncertainty-method-ref", "name": "uncertainty_method_ref", "description": "Candidate answer field for: Which variability sources and uncertainty method apply to this score and sampling unit? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-sampling-unit", "name": "sampling_unit", "description": "Candidate answer field for: Which variability sources and uncertainty method apply to this score and sampling unit? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-variability-sources", "name": "variability_sources", "description": "Candidate answer field for: Which variability sources and uncertainty method apply to this score and sampling unit? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-interval-semantics", "name": "interval_semantics", "description": "Candidate answer field for: Which variability sources and uncertainty method apply to this score and sampling unit? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-replay-acceptance-rule", "name": "replay_acceptance_rule", "description": "Candidate answer field for: What evidence and tolerances distinguish successful replay from a merely similar result? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-tolerance-profile", "name": "tolerance_profile", "description": "Candidate answer field for: What evidence and tolerances distinguish successful replay from a merely similar result? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "replay-contract-data-reproducibility-limitations", "name": "reproducibility_limitations", "description": "Candidate answer field for: What evidence and tolerances distinguish successful replay from a merely similar result? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] } ], "artifacts": [ { "id": "replay-contract-artifact", "name": "Replay and uncertainty specification", "description": "Versioned definition or evidence-index artifact for replay and uncertainty contract. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-006" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-assurance", "name": "Assurance and change", "description": "Qualification, comparison and retirement of the definition.", "rationale": "Groups related procedure-definition concerns while preserving the external evaluation lifecycle.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-005", "SRC-008" ], "layers": [ { "id": "layer-qualification", "name": "Verification and comparison", "description": "Context for verification and comparison within a versioned scoring procedure.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004", "SRC-005" ], "findings": [ { "id": "conformance-plan", "name": "Procedure verification and negative fixtures", "description": "A qualification plan tests the scorer definition and bindings before use. Local research schema validation is not evidence that any deployed scorer implements them. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ], "questions": [ { "id": "conformance-plan-q1", "text": "Which known-answer fixtures verify formulas, parsers and aggregation behavior?", "kind": "validation", "answer_data": [ "fixture_suite_ref", "expected_result_rules", "numeric_tolerances" ] }, { "id": "conformance-plan-q2", "text": "Which malformed, empty, duplicated and adversarial input classes must be rejected or qualified?", "kind": "quality", "answer_data": [ "negative_fixture_classes", "expected_failure_states" ] }, { "id": "conformance-plan-q3", "text": "Which implementation revision and reviewer evidence support a conformance assertion?", "kind": "evidence", "answer_data": [ "implementation_ref", "conformance_report_ref", "reviewer_ref" ] } ], "data_elements": [ { "id": "conformance-plan-data-fixture-suite-ref", "name": "fixture_suite_ref", "description": "Candidate answer field for: Which known-answer fixtures verify formulas, parsers and aggregation behavior? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "conformance-plan-data-expected-result-rules", "name": "expected_result_rules", "description": "Candidate answer field for: Which known-answer fixtures verify formulas, parsers and aggregation behavior? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "conformance-plan-data-numeric-tolerances", "name": "numeric_tolerances", "description": "Candidate answer field for: Which known-answer fixtures verify formulas, parsers and aggregation behavior? Instance constraints must be specified in the adopting profile.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "conformance-plan-data-negative-fixture-classes", "name": "negative_fixture_classes", "description": "Candidate answer field for: Which malformed, empty, duplicated and adversarial input classes must be rejected or qualified? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "conformance-plan-data-expected-failure-states", "name": "expected_failure_states", "description": "Candidate answer field for: Which malformed, empty, duplicated and adversarial input classes must be rejected or qualified? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "conformance-plan-data-implementation-ref", "name": "implementation_ref", "description": "Candidate answer field for: Which implementation revision and reviewer evidence support a conformance assertion? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "conformance-plan-data-conformance-report-ref", "name": "conformance_report_ref", "description": "Candidate answer field for: Which implementation revision and reviewer evidence support a conformance assertion? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] }, { "id": "conformance-plan-data-reviewer-ref", "name": "reviewer_ref", "description": "Candidate answer field for: Which implementation revision and reviewer evidence support a conformance assertion? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] } ], "artifacts": [ { "id": "conformance-plan-artifact", "name": "Conformance plan and report index", "description": "Versioned definition or evidence-index artifact for procedure verification and negative fixtures. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005" ] } ], "inline_only_rationale": null }, { "id": "comparison-policy", "name": "Change impact and comparability", "description": "Procedure revisions are compared by declared semantic differences, not by shared display names. The evaluation owner decides whether historical results need a fresh run. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ], "questions": [ { "id": "comparison-policy-q1", "text": "Which changes create a new semantic revision rather than a metadata correction?", "kind": "lifecycle", "answer_data": [ "revision_policy", "semantic_change_classes" ] }, { "id": "comparison-policy-q2", "text": "Which bindings must match before scores from two revisions can be compared?", "kind": "relationship", "answer_data": [ "comparability_dimensions", "matched_binding_rule", "comparison_limitations" ] }, { "id": "comparison-policy-q3", "text": "Who determines whether a changed scorer requires re-evaluation of earlier results?", "kind": "decision", "answer_data": [ "impact_review_ref", "evaluation_owner_binding", "rerun_request_policy" ] } ], "data_elements": [ { "id": "comparison-policy-data-revision-policy", "name": "revision_policy", "description": "Candidate answer field for: Which changes create a new semantic revision rather than a metadata correction? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "comparison-policy-data-semantic-change-classes", "name": "semantic_change_classes", "description": "Candidate answer field for: Which changes create a new semantic revision rather than a metadata correction? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "comparison-policy-data-comparability-dimensions", "name": "comparability_dimensions", "description": "Candidate answer field for: Which bindings must match before scores from two revisions can be compared? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "comparison-policy-data-matched-binding-rule", "name": "matched_binding_rule", "description": "Candidate answer field for: Which bindings must match before scores from two revisions can be compared? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "comparison-policy-data-comparison-limitations", "name": "comparison_limitations", "description": "Candidate answer field for: Which bindings must match before scores from two revisions can be compared? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "comparison-policy-data-impact-review-ref", "name": "impact_review_ref", "description": "Candidate answer field for: Who determines whether a changed scorer requires re-evaluation of earlier results? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "comparison-policy-data-evaluation-owner-binding", "name": "evaluation_owner_binding", "description": "Candidate answer field for: Who determines whether a changed scorer requires re-evaluation of earlier results? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "comparison-policy-data-rerun-request-policy", "name": "rerun_request_policy", "description": "Candidate answer field for: Who determines whether a changed scorer requires re-evaluation of earlier results? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] } ], "artifacts": [ { "id": "comparison-policy-artifact", "name": "Change impact and comparison statement", "description": "Versioned definition or evidence-index artifact for change impact and comparability. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-retirement", "name": "Definition retirement", "description": "Context for definition retirement within a versioned scoring procedure.", "source_refs": [ "SRC-004", "SRC-008" ], "findings": [ { "id": "retirement-policy", "name": "Deprecation and governed disposal", "description": "The procedure can be withdrawn while historical references remain resolvable under the adopting retention policy. Confidential payload disposal is executed by its custodian; preservation is not perpetual. Proposed local profile design; cited sources ground the concern, not every implementation detail.", "source_refs": [ "SRC-004", "SRC-008" ], "questions": [ { "id": "retirement-policy-q1", "text": "Which conditions deprecate or withdraw this revision and identify a successor?", "kind": "lifecycle", "answer_data": [ "retirement_condition", "successor_ref", "effective_time" ] }, { "id": "retirement-policy-q2", "text": "Which local manifests, private rubrics and evidence copies may be retained or deleted?", "kind": "retention", "answer_data": [ "retention_policy_ref", "disposition_classes", "hold_rule" ] }, { "id": "retirement-policy-q3", "text": "What minimal tombstone remains after authorized disposal without exposing protected content?", "kind": "access", "answer_data": [ "tombstone_schema", "disclosure_rule", "disposal_authority_ref" ] } ], "data_elements": [ { "id": "retirement-policy-data-retirement-condition", "name": "retirement_condition", "description": "Candidate answer field for: Which conditions deprecate or withdraw this revision and identify a successor? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] }, { "id": "retirement-policy-data-successor-ref", "name": "successor_ref", "description": "Candidate answer field for: Which conditions deprecate or withdraw this revision and identify a successor? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] }, { "id": "retirement-policy-data-effective-time", "name": "effective_time", "description": "Candidate answer field for: Which conditions deprecate or withdraw this revision and identify a successor? Instance constraints must be specified in the adopting profile.", "value_kind": "timestamp", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] }, { "id": "retirement-policy-data-retention-policy-ref", "name": "retention_policy_ref", "description": "Candidate answer field for: Which local manifests, private rubrics and evidence copies may be retained or deleted? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] }, { "id": "retirement-policy-data-disposition-classes", "name": "disposition_classes", "description": "Candidate answer field for: Which local manifests, private rubrics and evidence copies may be retained or deleted? Instance constraints must be specified in the adopting profile.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] }, { "id": "retirement-policy-data-hold-rule", "name": "hold_rule", "description": "Candidate answer field for: Which local manifests, private rubrics and evidence copies may be retained or deleted? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] }, { "id": "retirement-policy-data-tombstone-schema", "name": "tombstone_schema", "description": "Candidate answer field for: What minimal tombstone remains after authorized disposal without exposing protected content? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] }, { "id": "retirement-policy-data-disclosure-rule", "name": "disclosure_rule", "description": "Candidate answer field for: What minimal tombstone remains after authorized disposal without exposing protected content? Instance constraints must be specified in the adopting profile.", "value_kind": "text", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] }, { "id": "retirement-policy-data-disposal-authority-ref", "name": "disposal_authority_ref", "description": "Candidate answer field for: What minimal tombstone remains after authorized disposal without exposing protected content? Instance constraints must be specified in the adopting profile.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-008" ] } ], "artifacts": [ { "id": "retirement-policy-artifact", "name": "Deprecation and disposition record", "description": "Versioned definition or evidence-index artifact for deprecation and governed disposal. External runtime results and dataset payloads remain in their master systems.", "media_or_form": [ "structured record", "human-readable projection" ], "serial": true, "identity_strategy": "Authoritative master-system identifier first; governed IRI next; otherwise Dimension UUID or ULID. Preserve revision and digest separately; dates and filenames are not identity.", "source_refs": [ "SRC-004", "SRC-008" ] } ], "inline_only_rationale": null } ] } ] } ] }, "functions": [ { "id": "function-validate", "name": "Validate a procedure definition", "description": "Proposed, unimplemented local operation. Check references, applicable modes, formula domains and required failure policies.", "inputs": [ "procedure revision", "adopting profile" ], "outputs": [ "validation report with unresolved conditions" ], "preconditions": [ "Authorized role and current expected revision are resolved.", "Referenced schemas and policies are available or explicitly unknown." ], "effects": [ "Records a local validation observation; does not certify scorer accuracy.", "No external invocation, deployment, evaluation approval or dataset mutation is performed." ], "source_refs": [ "SRC-001", "SRC-005" ] }, { "id": "function-bind", "name": "Prepare a scoring request", "description": "Proposed, unimplemented local operation. Resolve permitted item, material and judge bindings into a request manifest.", "inputs": [ "procedure revision", "external evaluation reference", "input references" ], "outputs": [ "request manifest or refusal with missing preconditions" ], "preconditions": [ "Authorized role and current expected revision are resolved.", "Material access, scope and revision pins are checked." ], "effects": [ "Creates only a local request proposal for the external runtime.", "No external invocation, deployment, evaluation approval or dataset mutation is performed." ], "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "function-inspect", "name": "Inspect a supplied score envelope", "description": "Proposed, unimplemented local operation. Check structure, status, provenance links and denominator disclosures against the chosen revision.", "inputs": [ "supplied result envelope", "procedure revision" ], "outputs": [ "qualified inspection report or rejection" ], "preconditions": [ "Authorized role and current expected revision are resolved.", "Supplied content is handled as untrusted data and is not executed." ], "effects": [ "Keeps original verdict intact; returns validation evidence to its evaluation owner.", "No external invocation, deployment, evaluation approval or dataset mutation is performed." ], "source_refs": [ "SRC-004", "SRC-005" ] }, { "id": "function-qualify", "name": "Record qualification evidence", "description": "Proposed, unimplemented local operation. Attach reviewed fixture and judge-calibration evidence to the exact procedure revision.", "inputs": [ "evidence references", "reviewer decision", "procedure revision" ], "outputs": [ "qualification record with limitations or refusal" ], "preconditions": [ "Authorized role and current expected revision are resolved.", "Evidence identity, scope, conflicts and reviewer authority are recorded." ], "effects": [ "Records definition qualification only, not an evaluated-model release approval.", "No external invocation, deployment, evaluation approval or dataset mutation is performed." ], "source_refs": [ "SRC-001", "SRC-002", "SRC-003" ] }, { "id": "function-compare", "name": "Assess revision comparability", "description": "Proposed, unimplemented local operation. Produce a semantic difference and comparability statement for a pair of definitions.", "inputs": [ "two pinned procedure revisions", "comparison profile" ], "outputs": [ "change impact report and unresolved differences" ], "preconditions": [ "Authorized role and current expected revision are resolved.", "Task, material, metric, rubric and runtime bindings are available." ], "effects": [ "Suggests re-evaluation for the evaluation owner; never rewrites historical results.", "No external invocation, deployment, evaluation approval or dataset mutation is performed." ], "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "function-retire", "name": "Record procedure retirement", "description": "Proposed, unimplemented local operation. Record authorized deprecation, successor and disposition references.", "inputs": [ "procedure revision", "authority record", "retention policy" ], "outputs": [ "retirement record or refusal" ], "preconditions": [ "Authorized role and current expected revision are resolved.", "Retention holds, dependent references and disclosure restrictions are checked." ], "effects": [ "Marks the definition retired locally; payload disposal is delegated to the adopting custodian.", "No external invocation, deployment, evaluation approval or dataset mutation is performed." ], "source_refs": [ "SRC-004", "SRC-008" ] } ], "composition": [ { "target": "WM-AI-003", "relation": "REFERENCE", "purpose": "Proposed binding to the evaluation owner. Registry parent is reconciled as context, not automatic inheritance; evaluation runs, results, findings and approvals remain there.", "required": false, "source_refs": [ "SRC-004", "SRC-005" ] }, { "target": "W3C PROV-O", "relation": "ALIGN", "purpose": "Map procedure revisions to entities/plans and external invocation evidence to activities and attribution. No implemented mapping asserted.", "required": false, "source_refs": [ "SRC-004" ] }, { "target": "W3C EARL 1.0 Schema", "relation": "ALIGN", "purpose": "Candidate mapping of criterion, assertor, subject and outcome in the response envelope. Note status and lossy mappings must remain visible.", "required": false, "source_refs": [ "SRC-005" ] }, { "target": "RFC 8785", "relation": "ALIGN", "purpose": "Optional JSON canonicalization binding for integrity checks, with no semantic or measurement conformance claim.", "required": false, "source_refs": [ "SRC-006" ] } ], "serviceLayers": { "dimension": { "owner_package_requirements": [ "Name an accountable procedure custodian, namespace authority and external evaluation owner using role references, never a vendor owner.", "Pin specification, storage, interface and process bindings with access and retention policy references.", "Declare applicability, evidence trust, revision compatibility and unresolved operational dependencies before use." ], "namespace_guidance": "Use a governed procedure namespace and stable local IDs. Store logical identity, semantic revision and representation digest separately.", "registry_links": [ "vr.wm-ai-011", "WM-AI-003 - proposed evaluation reference; registry parent is not inheritance" ] }, "canon_and_patch": { "canonicalization_rules": [ "Keep one authoritative definition per identity and semantic revision. JSON, YAML and other projections preserve logical meaning.", "An RFC 8785 binding may canonicalize eligible JSON bytes; use explicit number and Unicode rules and never claim that generic sorted JSON implements JCS." ], "patch_rules": [ "Patch with author, reason, source evidence, expected revision and review record. Scoring, rubric, preprocessing or aggregation changes create new semantic revisions.", "Metadata corrections are traceable; disputed qualification records are superseded rather than silently replaced." ], "compatibility_rules": [ "Require an explicit comparison profile before pooling results across revisions; identical names are insufficient.", "Retain interpretable historical references subject to lawful disposition; unavailable dependencies restrict replay claims." ] }, "artifact_rules": { "identity_priority": [ "Authoritative master-system identifier", "Governed global identifier or IRI", "UUID or ULID assigned by the adopting Dimension" ], "timestamp_rule": "Use RFC 3339 timestamps with seconds and explicit offset or Z. Separate event time from observation and ingestion time; date-only policy values remain date-only.", "serial_naming_rule": "Use stable procedure and artifact IDs plus independent revision fields; filenames and dates are display labels only.", "integrity_rule": "Record digest algorithm and exact byte domain for retained representations. Verify reference and access integrity separately; a digest proves neither measurement validity nor authority. Minimize sensitive content in evidence indexes." }, "policies": [ "All structure and local operations are proposed research design. No implemented scorer, universal benchmark, external conformance or safety certification is claimed.", "Treat reference text, evaluated output and judge rationale as untrusted data; adopt least authority and validate structured outputs.", "Keep high-risk subject evaluations at policy level and obtain qualified scope review before operational use.", "Do not treat an unknown, errored or omitted result as a successful score. Preserve denominator and comparability limitations.", "Separate rubric qualification, evaluated-model approval and data access authority." ], "crud": { "read": [ "Resolve identity, revision, holds and access policy before reading definitions or evidence pointers." ], "create": [ "Create a draft definition with custodian and scope; bind material and judge dependencies without copying their masters." ], "update": [ "Require expected revision and authority; append reviewed semantic revisions and retain qualified supersession links." ], "delete": [ "Apply the adopting Dimension retention and disposition policy to local manifests, evidence copies and private rubric payloads. Check holds and dependencies; the adopting custodian executes deletion.", "Retain only permitted minimal tombstones and provenance, redact protected payloads when required, and never delete externally mastered datasets, evaluation runs or results through this model." ] }, "roles": [ { "name": "Procedure custodian", "responsibilities": [ "Maintains definitions and proposes revisions within delegated authority." ] }, { "name": "Measurement reviewer", "responsibilities": [ "Assesses construct validity, calibration and comparability evidence." ] }, { "name": "Evaluation owner", "responsibilities": [ "Owns the external evaluation lifecycle and decisions based on results." ] }, { "name": "Runtime operator", "responsibilities": [ "Implements invocation policy and records actual runtime events externally." ] }, { "name": "Data custodian", "responsibilities": [ "Controls protected material, retention and authorized disposal." ] } ], "access": { "default_rule": "Deny ungranted access; public definition metadata does not grant access to labels, private prompts or run payloads.", "scopes": [ "bundle", "layer", "finding", "artifact" ], "exceptions": [ "Time-limited reviewer access requires purpose, authority and an audit reference; it never grants execution authority." ], "audit_requirements": [ "Reference the external audit trail for access and change events, with actor, reason, revision and event time. Do not create a competing audit lifecycle.", "Record disclosure, redaction and disposition decisions with minimal personal data." ] }, "agents_bootstrap": { "filename": "AGENTS.md", "required_fields": [ "Name", "Type", "Specification URL", "Storage type URL", "Interface URL", "Processes URL" ], "read_order": [ "AGENTS.md", "specification and visible research holds", "adopting Dimension policies and pinned procedure revision", "external evaluation, material and runtime bindings" ] } }, "coverage": { "claim": "Source-grounded proposed model of one versioned evaluation harness and scoring procedure, including optional model judges. A separate frozen local no-tools self-audit found no critical contradictions. This remains a reviewable draft with source-verification, independent-review, profile qualification and executable-conformance holds; no operational scorer or universal coverage is claimed.", "confidence": "medium", "checklist": [ { "dimension": "identity", "status": "covered", "notes": "Master identity and semantic revisions are distinct from invocation IDs." }, { "dimension": "lifecycle", "status": "covered", "notes": "Draft, qualification, supersession and retirement apply to procedure definitions only." }, { "dimension": "relationships", "status": "covered", "notes": "Parent and externally mastered dataset, model and runtime boundaries are explicit." }, { "dimension": "temporal", "status": "covered", "notes": "Effective, event, observation and ingestion roles are separated." }, { "dimension": "provenance", "status": "covered", "notes": "Reference revisions, derivation and evidence pointers are retained with trust limits." }, { "dimension": "ownership", "status": "covered", "notes": "Custodian, measurement reviewer, evaluation owner and runtime operator are distinct roles." }, { "dimension": "validation", "status": "covered", "notes": "Definition and fixture qualification are specified; implementation tests remain a gap." }, { "dimension": "access", "status": "covered", "notes": "Held-out labels, private rubrics and sensitive evidence are scope-controlled." }, { "dimension": "retention and deletion", "status": "covered", "notes": "Local record disposition and minimal tombstones follow the adopting policy." }, { "dimension": "interoperability", "status": "covered", "notes": "Conceptual PROV-O, EARL and optional JSON canonicalization mappings are qualified." }, { "dimension": "physical properties", "status": "not-applicable", "notes": "The root is a procedure definition; no mass, location geometry or physical actuation is intrinsic." }, { "dimension": "measurement and uncertainty", "status": "covered", "notes": "Formula, scale, denominator and variability questions require profile-specific answers." }, { "dimension": "model judge qualification", "status": "covered", "notes": "Rubric, calibration, sensitivity, abstention and trust requirements are represented." }, { "dimension": "executable conformance", "status": "gap", "notes": "No instance schema, runtime binding, calibration experiment or executable scoring fixtures delivered." }, { "dimension": "source verification", "status": "gap", "notes": "Selected web content was reviewed; direct HTTP checks and current-version verification remain unperformed." }, { "dimension": "independent review", "status": "gap", "notes": "External providers skipped; local no-tools self-audit is not independent review." } ], "known_omissions": [ "Nested instance constraints, operational scorer implementations and executable fixtures are not delivered.", "Statistical estimator choice, sample-size planning, domain-specific validity and human reference-label protocols require qualified profiles.", "Exact model, dataset, audit and runtime neighbor IDs and versions remain unpinned.", "Multimodal, interactive, embodied and high-risk use cases require specialized profiles; this draft offers no operational evaluation recipes.", "Live-source version checks, reproduction rights and independent external review remain incomplete." ], "conflicts": [], "regional_assumptions": [ "No jurisdiction-specific legal compliance is asserted. Adopters determine privacy, rights, retention and review requirements.", "Public guidance is used for conceptual design, not as a universal obligation or institutional endorsement." ], "adversarial_checks": [ "A judge response that matches its schema can still be false or biased.", "An errored or untested item cannot silently reduce the denominator or count as passing.", "Seed equality and matching digests cannot establish cross-environment replay.", "A changed rubric invalidates silent comparisons with earlier scores.", "A parent registry entry cannot transfer evaluation approvals or dataset ownership to this procedure.", "Traceable retention cannot be interpreted as indefinite retention of private payloads." ] }, "researchAdjudication": { "providerMode": "single-provider-waiver", "activeProviders": [ "codex" ], "waivedProviders": [ "claude", "grok" ], "providerPolicy": { "contract_version": "1.0.0", "mode": "single-provider-waiver", "effective_at": "2026-09-06T00:00:00Z", "scope": "Canonical single-stream subject-model research after the six-workstream consolidation", "active_providers": [ "codex" ], "waived_providers": [ { "provider": "claude", "authorized_by": "repository owner", "authorized_at": "2026-09-06T00:00:00Z", "reason": "Claude produced no result on prior 1800-second and 900-second attempts and again timed out on bounded 600-second Sonnet and 300-second Haiku passes. The owner prioritized completion over provider availability." }, { "provider": "grok", "authorized_by": "repository owner", "authorized_at": "2026-09-06T00:00:00Z", "reason": "The repository owner authorized completion without Grok when Grok is unavailable, slow or schema-invalid. Grok may still be attempted as a bounded supplemental reviewer, but its failure never blocks a valid Claude plus no-tools result." } ], "review_rule": "Codex may complete source-grounded fallback research after bounded Claude and Grok attempts fail. It requires a separate no-tools adversarial audit and remains reviewable-draft with a visible absence-of-external-review hold.", "supplemental_provider_attempts": [ { "provider": "claude", "required": false, "maximum_attempts": 1, "failure_policy": "record-and-continue", "admission_rule": "Use only a locally schema-valid result whose sources and boundaries survive adjudication." }, { "provider": "grok", "required": false, "maximum_attempts": 1, "failure_policy": "record-and-continue", "admission_rule": "Use only a locally schema-valid result whose sources and boundaries survive adjudication." } ] }, "boundaryDecision": { "entry_kind": "entity", "status": "accepted", "rationale": "The root is an identifiable procedure definition across immutable revisions and multiple invocations. A run is a separate activity and a reusable design pattern is not the instance root. The registry standalone-mm value classifies the record plane. WM-AI-003 remains the owner of evaluation runs, findings and approvals." }, "decisions": [ { "concept": "Procedure identity", "disposition": "accepted", "rationale": "Identity, revision, dependency digest and invocation are explicitly separate. PROV-O is a conceptual basis, not evidence that a deployed implementation exists." }, { "concept": "Parent and evaluation ownership", "disposition": "qualified", "rationale": "The registry parent does not establish inheritance or mandatory containment. With no incident ledger edges frozen, the proposed optional reference retains evaluation runs, findings and approvals outside this model." }, { "concept": "Input and material ownership", "disposition": "accepted", "rationale": "The input contract owns role bindings, admissibility and transformations while datasets, reference corpora and model artifacts remain externally mastered. Unknown contamination is not recast as absence of contamination." }, { "concept": "Scoring and decision authority", "disposition": "separated", "rationale": "Formula, value domain, orientation, rubric and criterion thresholds describe score semantics. A criterion outcome does not grant release or deployment authority; those decisions remain with the evaluation owner." }, { "concept": "Missingness and denominator integrity", "disposition": "accepted", "rationale": "Aggregation questions retain eligible, attempted, valid and excluded counts and ask about duplicates and dependence. Failure, indeterminate, inapplicable and untested outcomes are not silently converted to valid values." }, { "concept": "Model judge correctness", "disposition": "qualified", "rationale": "A configured judge and structured verdict are not truth guarantees. The rubric and qualification findings preserve disputed reference labels, abstention, independent reference review and task-specific acceptance criteria." }, { "concept": "Judge bias evidence", "disposition": "deferred empirically", "rationale": "Order, verbosity, language and self-preference checks are proposed questions, not claimed experimental findings. The source pack motivates qualification but supplies no measured rates or universal test thresholds." }, { "concept": "Runtime and dangerous subject boundary", "disposition": "accepted at policy level", "rationale": "The definition states invocation, trust and resource constraints while external runtimes own execution. No exploit recipes or operational high-risk evaluation instructions occur; content cannot grant runtime authority." }, { "concept": "Result and audit ownership", "disposition": "accepted", "rationale": "Response schemas and evidence indexes belong to the definition; actual results and authoritative audit events remain with the evaluation and runtime masters. Local inspection preserves the original verdict." }, { "concept": "Reproducibility and uncertainty", "disposition": "qualified", "rationale": "Fixed seeds, identical names and representation hashes do not establish replay or comparability. The proposal asks for dependency pins, observation units, uncertainty methods and tolerances without inventing a statistical guarantee." }, { "concept": "Revision compatibility", "disposition": "accepted", "rationale": "Semantic changes require new procedure revisions and a comparison statement. Metadata correction, scoring change and re-evaluation decisions are separate; historical results are not overwritten." }, { "concept": "Retention and authority", "disposition": "accepted with adopting-policy dependency", "rationale": "Local manifests and private payloads follow scoped retention, holds and custodian disposal. Minimal tombstones do not require indefinite sensitive-data retention, and local retirement does not delete external runs." }, { "concept": "Open specification status", "disposition": "qualified", "rationale": "EARL is explicitly a Working Group Note and incomplete for procedure description. PROV-O mappings and RFC 8785 canonicalization are conceptual or optional bindings, not conformance evidence or measurement-validity proofs." }, { "concept": "Instance schemas and operations", "disposition": "deferred", "rationale": "Candidate data fields and six proposed local operations do not implement a scorer. Optional research fields need conditional requirements in an adopting schema; fixtures, calibration and runtime checks remain unexecuted." }, { "concept": "Source and provider assurance", "disposition": "held", "rationale": "Selected primary content was read through the web tool, but direct HTTP measurements were not attempted under the stated sandbox restriction. The frozen local self-audit adds no facts and cannot substitute for waived independent provider review." } ], "publicationHolds": [ "Independent external review is absent under the owner-authorized single-provider waiver. Claude and Grok were skipped with zero attempts; this separate local Codex no-tools self-audit is not independent second-provider review.", "Live source and current-version verification remains incomplete. Selected primary web content was reviewed, but direct HTTP checks were not attempted under the owner-declared sandbox restriction. There are zero measured HTTP responses, not evidence of source failure. The coordinator must run check_sources.py outside the sandbox and review version, claim-support and licensing evidence.", "Task, language, modality, statistical validity, uncertainty estimator, reference-label quality and model judge sensitivity require empirical profile qualification. Legal applicability, privacy, retention and high-risk use require qualified adopting-policy review; no compliance or safety certification is claimed.", "Nested instance schemas, pinned neighbor and runtime bindings, executable scorer fixtures and PROV-O/EARL mappings remain incomplete. No implementation, benchmark run, calibration experiment or conformance certification has been delivered.", "Independent external review was explicitly waived by the repository owner; this codex-only result remains a reviewable draft." ], "deferredResearch": [ "Restore independent external review and verify admitted source versions and claim support before any canonical or publishable-draft promotion.", "Specify conditional instance constraints and executable fixtures for input joins, malformed responses, empty denominators, retries, semantic changes and lawful disposition.", "Empirically qualify judge calibration, bias sensitivity, reference-label protocols, uncertainty methods and comparability for each intended task profile.", "Pin external model, dataset, evaluation, audit and runtime bindings and test loss-aware interoperability mappings." ] }, "statistics": { "sources": 8, "bundles": 6, "layers": 12, "findings": 18, "questions": 54, "artifacts": 18, "functions": 6 } }