# Vercy AI instruction - YAML 1.2 (JSON-compatible) { "vercy": "1.0-draft", "publication": { "status": "published", "adjudicationStatus": "reviewable-draft", "publishableCanonical": false, "generatedAt": "2026-10-21T17:40:37Z", "synthesisSha256": "85356431e05cbe3492a2e0423463ecc6911450aee232acb1bb8e351b432873d7", "providerMode": "single-provider-waiver", "providers": [ "Codex" ], "waivedProviders": [ "Claude", "Grok" ] }, "metaModel": { "id": "WM-AI-013", "registryId": "vr.wm-ai-013", "name": "Ablation Study", "version": "0.1.0", "previousVersions": [], "entryKind": "entity", "family": "World Models", "category": "Information and virtual systems", "industry": [ "Cross-industry" ], "domain": [ "INF.AI.ABL" ], "tags": [ "ablation", "study", "inf.ai.abl" ], "status": "published" }, "canonicalUrl": "https://ver.cy/models/wm-ai-013-ablation-study/", "sourceUrl": "https://github.com/ver-cy/world-models/tree/feat/mega-model-registry/research/runs/wm-ai-013", "model": { "registry_id": "vr.wm-ai-013", "model_id": "WM-AI-013", "name": "Ablation Study", "entry_kind": "entity", "purpose": "Record controlled with-factor and without-factor comparisons on a common evaluation corpus using one scorer contract per contrast.", "scope_statement": "One persistent, versioned ablation study record owns its protocol, arm comparison intent, evidence references and bounded interpretations. Runs, datasets, scorer implementations and benchmark result masters remain external. This is a proposed research specification, not an executed experiment.", "in_scope": [ "Study identity, hypothesis, intervention arms and protocol revisions", "Common evaluation cases, scorer contract, controlled changes and deviations", "Run references, failures, paired observations, effect estimates and uncertainty", "Claim qualification, review, evidence access and retention" ], "out_of_scope": [ "Executing training, inference, scoring or deployment", "Mastering datasets, labels, model weights, scorer code or general benchmark leaderboards", "Universal causal identification, statistical test selection or safety certification", "Operational instructions for harmful evaluations" ], "boundary_notes": [ { "neighbor": "WM-AI-003 AI Model Evaluation", "distinction": "Registry parent is a conceptual specialization lead. The study adds controlled intervention contrasts; no unverified runtime inheritance or containment is asserted.", "source_refs": [ "SRC-001", "SRC-003" ] }, { "neighbor": "WM-AI-004 AI Inference / Agent Run and WM-AI-006 Model Training / Fine-tuning Run", "distinction": "Execution attempts have separate masters. A study can exist before runs and may reference optional training runs when retraining is part of its intervention.", "source_refs": [ "SRC-005", "SRC-008" ] }, { "neighbor": "WM-AI-009 Evaluation Dataset / Benchmark and WM-AI-014 Ground-truth Label Set", "distinction": "Frozen case and label references are used; dataset curation and label lifecycle are not owned here.", "source_refs": [ "SRC-005", "SRC-006" ] }, { "neighbor": "WM-AI-011 Evaluation Harness and Scorer and WM-AI-012 Benchmark Result", "distinction": "Scorer definitions and result records remain external. The study owns the rationale for comparing arms, not the master scores or implementation.", "source_refs": [ "SRC-002", "SRC-005" ] }, { "neighbor": "Reusable experiment design and factorial experiment", "distinction": "This root is an instantiated study record, not a design pattern. Multi-arm contrasts are allowed with declared settings; a single ablation cannot isolate all interactions.", "source_refs": [ "SRC-001", "SRC-003" ] } ] }, "sources": [ { "id": "SRC-001", "title": "One variable at a time", "organization": "NIST", "url": "https://www.itl.nist.gov/div898/handbook/pri/section2/pri212.htm", "version_or_date": "Undated handbook section 5.2.1.2; version not pinned", "source_type": "public-authority", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T17:39:13Z", "relevance": "Single-factor comparisons do not identify interactions. Used to bound attribution, not prescribe an AI schema." }, { "id": "SRC-002", "title": "Two-Sample t-Test for Equal Means", "organization": "NIST", "url": "https://www.itl.nist.gov/div898/handbook/eda/section3/eda353.htm", "version_or_date": "Undated handbook section 1.3.5.3; version not pinned", "source_type": "public-authority", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T17:39:13Z", "relevance": "Defines paired observations and differences; does not justify a universal test for every metric." }, { "id": "SRC-003", "title": "How do you select an experimental design?", "organization": "NIST", "url": "https://www.itl.nist.gov/div898/handbook/pri/section3/pri33.htm", "version_or_date": "Undated handbook section 5.3.3; version not pinned", "source_type": "public-authority", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T17:39:13Z", "relevance": "Experimental objectives, factors and resources inform design selection." }, { "id": "SRC-004", "title": "Randomized block designs", "organization": "NIST", "url": "https://www.itl.nist.gov/div898/handbook/pri/section3/pri332.htm", "version_or_date": "Undated handbook section 5.3.3.2; version not pinned", "source_type": "public-authority", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T17:39:13Z", "relevance": "Nuisance factors, blocks and randomization motivate explicit control records." }, { "id": "SRC-005", "title": "PROV-O: The PROV Ontology", "organization": "W3C", "url": "https://www.w3.org/TR/prov-o/", "version_or_date": "W3C Recommendation 2013-04-30", "source_type": "ontology", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T17:39:13Z", "relevance": "Entities, activities, agents and derivations support traceable study evidence; they do not prove experimental validity." }, { "id": "SRC-006", "title": "RO-Crate 1.2 Introduction", "organization": "RO-Crate community", "url": "https://www.researchobject.org/ro-crate/specification/1.2/introduction.html", "version_or_date": "1.2 selected profile; no latest-version claim", "source_type": "schema", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T17:39:13Z", "relevance": "Metadata aggregation and contextual descriptions motivate portable evidence packages. This proposal is not an implemented RO-Crate profile." }, { "id": "SRC-007", "title": "Statistical Comparisons of Classifiers over Multiple Data Sets", "organization": "Janez Demsar, research author", "url": "https://www.jmlr.org/papers/v7/demsar06a.html", "version_or_date": "JMLR 7, 1-30, 2006; abstract reviewed", "source_type": "scientific", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T17:39:13Z", "relevance": "The primary research abstract concerns comparisons across multiple datasets. It motivates a scope warning, not automatic use on per-item scores within one corpus." }, { "id": "SRC-008", "title": "NeurIPS Paper Checklist Guidelines", "organization": "NeurIPS research community", "url": "https://neurips.cc/public/guides/PaperChecklist", "version_or_date": "Living guidelines accessed 2026-10-21; no immutable version", "source_type": "first-party-doc", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T17:39:13Z", "relevance": "Research reporting guidance on reproducibility, experimental details, variability, resources and limitations. Selected sections reviewed, not a compliance certification." } ], "structure": { "bundles": [ { "id": "bundle-study", "name": "Study charter and protocol", "description": "Identify the study and preserve its declared intent.", "rationale": "Original research structure grouping related study responsibilities; cited sources ground principles, not these node names.", "source_refs": [ "SRC-003", "SRC-005", "SRC-008" ], "layers": [ { "id": "layer-identity", "name": "Study identity", "description": "Proposed record obligations for study identity.", "source_refs": [ "SRC-003", "SRC-005" ], "findings": [ { "id": "finding-identity", "name": "Study identity and accountable scope", "description": "One persistent study record binds its purpose and accountable role across protocol revisions. A run, report or reusable experimental recipe has a different identity.", "source_refs": [ "SRC-003", "SRC-005" ], "questions": [ { "id": "finding-identity-q01", "kind": "identity", "text": "Which authoritative study identifier and revision distinguish this study from a rerun or replication?", "answer_data": [ "Master namespace, study ID, revision ID, replication link" ] }, { "id": "finding-identity-q02", "kind": "ownership", "text": "Which accountable study role owns the record and which evidence custodian maintains its references?", "answer_data": [ "Role identifiers, delegated scope, custody contact" ] }, { "id": "finding-identity-q03", "kind": "definition", "text": "What precise factor contribution is being investigated on which target population?", "answer_data": [ "Hypothesis, factor, target population, intended interpretation" ] }, { "id": "finding-identity-q04", "kind": "classification", "text": "Is this a planned, exploratory or confirmatory study and how was that classification determined?", "answer_data": [ "Study class, classification basis, decision time" ] } ], "data_elements": [ { "id": "finding-identity-data01", "name": "study_identity", "description": "Authoritative namespace and stable study ID, independent of timestamps.", "value_kind": "identifier", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "finding-identity-data02", "name": "accountability", "description": "Accountable role and evidence custodian references; no fixed company owner.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "finding-identity-data03", "name": "objective", "description": "Hypothesis, population, classification and classification rationale.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005" ] } ], "artifacts": [ { "id": "finding-identity-artifact", "name": "Study charter", "description": "Versioned evidence for study identity and accountable scope. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-003", "SRC-005" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-protocol", "name": "Protocol and revision", "description": "Proposed record obligations for protocol and revision.", "source_refs": [ "SRC-003", "SRC-005", "SRC-008" ], "findings": [ { "id": "finding-protocol", "name": "Frozen comparison intent", "description": "A frozen protocol revision states intended contrasts, metrics and decision rules. Amendments remain visible and post-result choices cannot be relabelled as prespecified.", "source_refs": [ "SRC-003", "SRC-005", "SRC-008" ], "questions": [ { "id": "finding-protocol-q01", "kind": "temporal", "text": "When were hypotheses and analysis rules frozen relative to the first outcome inspection?", "answer_data": [ "Freeze time, first inspection time, timing uncertainty" ] }, { "id": "finding-protocol-q02", "kind": "lifecycle", "text": "Which transitions move this study from design through execution, analysis and review to closure or withdrawal?", "answer_data": [ "Current state, transition evidence, actor and reason" ] }, { "id": "finding-protocol-q03", "kind": "event", "text": "Which protocol amendments occurred and which comparisons must be repeated or reclassified?", "answer_data": [ "Amendment ID, prior revision, affected contrasts, reclassification" ] }, { "id": "finding-protocol-q04", "kind": "authority", "text": "Who may freeze or amend the protocol and what review is required for an outcome-informed change?", "answer_data": [ "Role scope, review decision, evidence reference" ] } ], "data_elements": [ { "id": "finding-protocol-data01", "name": "protocol_ref", "description": "Immutable protocol revision and content digest.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005", "SRC-008" ] }, { "id": "finding-protocol-data02", "name": "transition_log", "description": "State transitions, amendments and affected results.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005", "SRC-008" ] }, { "id": "finding-protocol-data03", "name": "freeze_time", "description": "Recorded freeze time with seconds and offset; unknown stays explicit.", "value_kind": "timestamp", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-003", "SRC-005", "SRC-008" ] } ], "artifacts": [ { "id": "finding-protocol-artifact", "name": "Protocol revision ledger", "description": "Versioned evidence for frozen comparison intent. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-003", "SRC-005", "SRC-008" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-design", "name": "Interventions and controls", "description": "Make arm changes and conditional comparison boundaries inspectable.", "rationale": "Original research structure grouping related study responsibilities; cited sources ground principles, not these node names.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004" ], "layers": [ { "id": "layer-intervention", "name": "Factor intervention", "description": "Proposed record obligations for factor intervention.", "source_refs": [ "SRC-001", "SRC-003" ], "findings": [ { "id": "finding-intervention", "name": "With-factor and without-factor arms", "description": "Each contrast names a reference arm and an ablated arm. Removal, masking or replacement semantics are explicit; training-time changes and inference-time disabling answer different questions. Incidental changes limit attribution.", "source_refs": [ "SRC-001", "SRC-003" ], "questions": [ { "id": "finding-intervention-q01", "kind": "composition", "text": "Which component or configuration factor is removed, masked or replaced in each arm?", "answer_data": [ "Factor selector, reference value, intervention value, arm IDs" ] }, { "id": "finding-intervention-q02", "kind": "process", "text": "Does the intervention require retraining or only alter inference and what remains fixed?", "answer_data": [ "Intervention phase, training policy, invariant manifest" ] }, { "id": "finding-intervention-q03", "kind": "validation", "text": "How is actual intervention compliance checked against intended arm configurations?", "answer_data": [ "Configuration diff, verification evidence, unintended changes" ] }, { "id": "finding-intervention-q04", "kind": "exception", "text": "What happens when removing the factor makes an arm non-executable or changes its interface?", "answer_data": [ "Invalid-arm status, compatibility adaptation, exclusion rationale" ] } ], "data_elements": [ { "id": "finding-intervention-data01", "name": "arm_manifest", "description": "Arm IDs, immutable configuration references and factor settings.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-003" ] }, { "id": "finding-intervention-data02", "name": "intervention", "description": "Removal or substitution semantics and training versus inference phase.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-003" ] }, { "id": "finding-intervention-data03", "name": "compliance", "description": "Observed configuration differences, deviations and admissibility decision.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-003" ] } ], "artifacts": [ { "id": "finding-intervention-artifact", "name": "Arm and intervention manifest", "description": "Versioned evidence for with-factor and without-factor arms. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-001", "SRC-003" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-controls", "name": "Experimental controls", "description": "Proposed record obligations for experimental controls.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004" ], "findings": [ { "id": "finding-controls", "name": "Confounders and interaction limits", "description": "The local design records nuisance variables, budget policy and blocks. A contrast at one background setting estimates only that conditional comparison; multiple removals do not automatically identify independent contributions.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004" ], "questions": [ { "id": "finding-controls-q01", "kind": "constraint", "text": "Which nuisance factors and execution budgets are controlled, blocked or left uncontrolled?", "answer_data": [ "Nuisance inventory, control method, budget policy, residual limitations" ] }, { "id": "finding-controls-q02", "kind": "process", "text": "How are run order and repeat allocation determined within declared blocks?", "answer_data": [ "Block definition, allocation rule, randomization record, exceptions" ] }, { "id": "finding-controls-q03", "kind": "relationship", "text": "Which other factor settings condition the contrast and which interactions are untested?", "answer_data": [ "Background settings, crossed contrasts, untested interactions" ] }, { "id": "finding-controls-q04", "kind": "quality", "text": "Could unequal tuning or resource allowances explain the observed difference?", "answer_data": [ "Tuning policy, allocated and observed resources, comparability verdict" ] } ], "data_elements": [ { "id": "finding-controls-data01", "name": "control_plan", "description": "Blocks, ordering, resource allowances and nuisance controls.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-003", "SRC-004" ] }, { "id": "finding-controls-data02", "name": "background_settings", "description": "Other factor levels and interaction coverage.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-003", "SRC-004" ] }, { "id": "finding-controls-data03", "name": "deviation_register", "description": "Uncontrolled changes with impact assessments.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-003", "SRC-004" ] } ], "artifacts": [ { "id": "finding-controls-artifact", "name": "Control and deviation register", "description": "Versioned evidence for confounders and interaction limits. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-evaluation", "name": "Common evaluation contract", "description": "Bind the shared corpus and scorer before admitting comparisons.", "rationale": "Original research structure grouping related study responsibilities; cited sources ground principles, not these node names.", "source_refs": [ "SRC-002", "SRC-005", "SRC-006", "SRC-008" ], "layers": [ { "id": "layer-corpus", "name": "Corpus binding", "description": "Proposed record obligations for corpus binding.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ], "findings": [ { "id": "finding-corpus", "name": "Common evaluation cases and labels", "description": "Both arms reference the same frozen evaluation case set, split and label revision for a contrast. Input transformation may be the declared intervention while case identity remains stable. Membership drift and training overlap are recorded as threats.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ], "questions": [ { "id": "finding-corpus-q01", "kind": "identity", "text": "Which dataset revision, split and ordered or keyed case membership are common to both arms?", "answer_data": [ "Dataset ID, revision, split, case membership digest" ] }, { "id": "finding-corpus-q02", "kind": "provenance", "text": "Which label revision and preprocessing lineage support each evaluation case?", "answer_data": [ "Label set reference, preprocessing references, origin and revisions" ] }, { "id": "finding-corpus-q03", "kind": "quality", "text": "What checks identify leakage, duplicate cases or exposure during tuning and training?", "answer_data": [ "Overlap checks, suspected contamination, unresolved access limits" ] }, { "id": "finding-corpus-q04", "kind": "exception", "text": "How are case omissions or label corrections handled without silently changing the paired cohort?", "answer_data": [ "Missing-case manifest, correction revision, rescoring requirement" ] } ], "data_elements": [ { "id": "finding-corpus-data01", "name": "corpus_binding", "description": "Case set, split, membership digest and pairing key.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] }, { "id": "finding-corpus-data02", "name": "label_binding", "description": "Pinned label set or explicit no-label applicability statement.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] }, { "id": "finding-corpus-data03", "name": "cohort_deviations", "description": "Omissions, duplicates, leakage evidence and corrections.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] } ], "artifacts": [ { "id": "finding-corpus-artifact", "name": "Evaluation cohort manifest", "description": "Versioned evidence for common evaluation cases and labels. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-scorer", "name": "Scoring contract", "description": "Proposed record obligations for scoring contract.", "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ], "findings": [ { "id": "finding-scorer", "name": "One scorer definition per contrast", "description": "A contrast binds one scorer version, rubric, metric direction and aggregation rule across arms. A stochastic scorer needs its own repeat policy. Changing a scorer creates a new scoring revision for both arms.", "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ], "questions": [ { "id": "finding-scorer-q01", "kind": "measurement", "text": "Which metric, units, preferred direction and aggregation define the comparison score?", "answer_data": [ "Metric definition, units, direction, aggregation and denominator" ] }, { "id": "finding-scorer-q02", "kind": "identity", "text": "Which harness, scorer code or rubric revision and configuration are shared by both arms?", "answer_data": [ "Harness ID, scorer revision, rubric digest, configuration" ] }, { "id": "finding-scorer-q03", "kind": "constraint", "text": "How are scorer randomness, blinding and order effects handled?", "answer_data": [ "Scorer repeat policy, blinding decision, ordering and calibration" ] }, { "id": "finding-scorer-q04", "kind": "validation", "text": "What evidence shows both arms were scored with the pinned scoring contract?", "answer_data": [ "Scoring receipt, version comparison, mismatch status" ] } ], "data_elements": [ { "id": "finding-scorer-data01", "name": "scorer_binding", "description": "Scorer identity, version, configuration and optional rubric reference.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ] }, { "id": "finding-scorer-data02", "name": "metric_contract", "description": "Metric units, direction, formula, weighting and denominator.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ] }, { "id": "finding-scorer-data03", "name": "scoring_checks", "description": "Scorer drift checks and stochastic evaluation policy.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ] } ], "artifacts": [ { "id": "finding-scorer-artifact", "name": "Scoring contract snapshot", "description": "Versioned evidence for one scorer definition per contrast. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-execution", "name": "Execution evidence", "description": "Connect actual attempts to complete, paired observations.", "rationale": "Original research structure grouping related study responsibilities; cited sources ground principles, not these node names.", "source_refs": [ "SRC-002", "SRC-004", "SRC-005", "SRC-008" ], "layers": [ { "id": "layer-runs", "name": "Run lineage", "description": "Proposed record obligations for run lineage.", "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ], "findings": [ { "id": "finding-runs", "name": "Execution realization and repeats", "description": "Execution remains externally mastered. The study indexes arm assignments, run attempts, environment snapshots and repetition units, preserving differences between a retry, an independent repeat and a reanalysis.", "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ], "questions": [ { "id": "finding-runs-q01", "kind": "relationship", "text": "Which external training, inference and evaluation runs realize each arm and repeat?", "answer_data": [ "Run references, arm ID, repeat ID, execution role" ] }, { "id": "finding-runs-q02", "kind": "provenance", "text": "Which environment, configuration, seed and dependency revisions were actually used?", "answer_data": [ "Execution environment manifest, seed policy, immutable digests" ] }, { "id": "finding-runs-q03", "kind": "temporal", "text": "When did each attempt start and end and which protocol revision governed it?", "answer_data": [ "Start/end timestamps, protocol reference, clock uncertainty" ] }, { "id": "finding-runs-q04", "kind": "classification", "text": "Which attempts are retries, independent repeats or rescoring of existing outputs?", "answer_data": [ "Attempt type, predecessor link, independence justification" ] } ], "data_elements": [ { "id": "finding-runs-data01", "name": "run_index", "description": "External run IDs, arm and repeat assignments and predecessor links.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "finding-runs-data02", "name": "environment_refs", "description": "Environment and configuration snapshots, seed and dependency references.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] }, { "id": "finding-runs-data03", "name": "execution_window", "description": "Start/end times, uncertainty and governing revision.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] } ], "artifacts": [ { "id": "finding-runs-artifact", "name": "Run lineage index", "description": "Versioned evidence for execution realization and repeats. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-004", "SRC-005", "SRC-008" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-observations", "name": "Paired observations", "description": "Proposed record obligations for paired observations.", "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ], "findings": [ { "id": "finding-observations", "name": "Complete outcomes and failures", "description": "Case and repeat keys join the two arms. Failures and absent results remain explicit; complete-case analysis must disclose exclusions and its population. A successful retry never overwrites a failed attempt.", "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ], "questions": [ { "id": "finding-observations-q01", "kind": "evidence", "text": "Which per-case or aggregate observations and missingness indicators back each arm score?", "answer_data": [ "Observation references, values, case/repeat keys, missingness status" ] }, { "id": "finding-observations-q02", "kind": "validation", "text": "Do pair keys join one-to-one and how are duplicates, unmatched cases and nonfinite scores rejected?", "answer_data": [ "Join audit, duplicate and missing counts, invalid-score report" ] }, { "id": "finding-observations-q03", "kind": "exception", "text": "Which failures, timeouts and exclusions occurred and how do they affect the denominator?", "answer_data": [ "Failure categories, exclusion reasons, denominator before/after" ] }, { "id": "finding-observations-q04", "kind": "retention", "text": "How are original attempts preserved when a retry or rescore produces a new observation?", "answer_data": [ "Prior observation reference, supersession link, retention policy" ] } ], "data_elements": [ { "id": "finding-observations-data01", "name": "outcome_index", "description": "Original observation references and case/repeat keys.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ] }, { "id": "finding-observations-data02", "name": "pairing_report", "description": "Matched, missing, duplicated and invalid counts.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ] }, { "id": "finding-observations-data03", "name": "failure_policy", "description": "Prespecified failure scoring or exclusion rules and actual deviations.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ] } ], "artifacts": [ { "id": "finding-observations-artifact", "name": "Paired outcome and failure ledger", "description": "Versioned evidence for complete outcomes and failures. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-002", "SRC-005", "SRC-008" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-analysis", "name": "Analysis and interpretation", "description": "Keep estimates, uncertainty and conclusions distinguishable.", "rationale": "Original research structure grouping related study responsibilities; cited sources ground principles, not these node names.", "source_refs": [ "SRC-001", "SRC-002", "SRC-007", "SRC-008" ], "layers": [ { "id": "layer-effect", "name": "Effect estimation", "description": "Proposed record obligations for effect estimation.", "source_refs": [ "SRC-002", "SRC-007", "SRC-008" ], "findings": [ { "id": "finding-effect", "name": "Signed comparison and uncertainty", "description": "The proposed convention is delta = score_with_factor - score_without_factor. Metric direction controls interpretation. Uncertainty identifies its sampling unit and method; dataset-level test guidance is not automatically applicable to item-level observations.", "source_refs": [ "SRC-002", "SRC-007", "SRC-008" ], "questions": [ { "id": "finding-effect-q01", "kind": "measurement", "text": "What signed difference and practical importance threshold are reported for each contrast?", "answer_data": [ "With/without scores, delta, metric direction, threshold and units" ] }, { "id": "finding-effect-q02", "kind": "quality", "text": "Which variability sources and sampling units support the uncertainty estimate?", "answer_data": [ "Sampling unit, repeat count, item clustering, variability sources" ] }, { "id": "finding-effect-q03", "kind": "process", "text": "Which interval or test procedure was selected and why are its assumptions appropriate?", "answer_data": [ "Method/version, assumptions, diagnostics, interval type and level" ] }, { "id": "finding-effect-q04", "kind": "exception", "text": "How are undefined relative changes, insufficient repeats and non-estimable intervals represented?", "answer_data": [ "Undefined reason, denominator status, unavailable-uncertainty marker" ] } ], "data_elements": [ { "id": "finding-effect-data01", "name": "contrast_estimate", "description": "Arm scores, signed delta, units, practical threshold and direction.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-007", "SRC-008" ] }, { "id": "finding-effect-data02", "name": "uncertainty", "description": "Method, interval, level, sampling unit, assumptions and missingness.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-007", "SRC-008" ] }, { "id": "finding-effect-data03", "name": "analysis_ref", "description": "Analysis code/configuration and input revision provenance.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-007", "SRC-008" ] } ], "artifacts": [ { "id": "finding-effect-artifact", "name": "Contrast estimate record", "description": "Versioned evidence for signed comparison and uncertainty. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-002", "SRC-007", "SRC-008" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-interpretation", "name": "Interpretation and robustness", "description": "Proposed record obligations for interpretation and robustness.", "source_refs": [ "SRC-001", "SRC-002", "SRC-007", "SRC-008" ], "findings": [ { "id": "finding-interpretation", "name": "Bounded conclusions and multiplicity", "description": "The study separates observed differences from explanatory claims and deployment decisions. Multiple contrasts, metrics or selected slices require a disclosed analysis family and selection history; a null result alone does not establish equivalence.", "source_refs": [ "SRC-001", "SRC-002", "SRC-007", "SRC-008" ], "questions": [ { "id": "finding-interpretation-q01", "kind": "decision", "text": "What conclusion is supported within this corpus, scorer and intervention setting?", "answer_data": [ "Conclusion, scope, evidence links, alternative explanations" ] }, { "id": "finding-interpretation-q02", "kind": "constraint", "text": "Which hypothesis family and stopping or selection rules bound multiplicity and selective reporting?", "answer_data": [ "Contrast family, adjustment policy, stopping rule, exploratory labels" ] }, { "id": "finding-interpretation-q03", "kind": "quality", "text": "Which sensitivity checks assess exclusion, scoring or seed choices and where do conclusions change?", "answer_data": [ "Sensitivity variants, estimate changes, instability flags" ] }, { "id": "finding-interpretation-q04", "kind": "requirement", "text": "What additional evidence is needed before claiming equivalence, causal isolation or wider generalization?", "answer_data": [ "Claim-specific assumptions, margins, missing checks, reviewer decision" ] } ], "data_elements": [ { "id": "finding-interpretation-data01", "name": "claim_register", "description": "Claims with explicit scope, evidence and alternatives.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-007", "SRC-008" ] }, { "id": "finding-interpretation-data02", "name": "analysis_family", "description": "Contrasts, metrics, multiplicity and selection history.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-007", "SRC-008" ] }, { "id": "finding-interpretation-data03", "name": "robustness", "description": "Sensitivity analyses and unresolved interpretation limits.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-007", "SRC-008" ] } ], "artifacts": [ { "id": "finding-interpretation-artifact", "name": "Interpretation and sensitivity report", "description": "Versioned evidence for bounded conclusions and multiplicity. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-001", "SRC-002", "SRC-007", "SRC-008" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-assurance", "name": "Review and evidence governance", "description": "Support review and controlled reuse of study evidence.", "rationale": "Original research structure grouping related study responsibilities; cited sources ground principles, not these node names.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ], "layers": [ { "id": "layer-review", "name": "Review and release", "description": "Proposed record obligations for review and release.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ], "findings": [ { "id": "finding-review", "name": "Evidence-backed review disposition", "description": "A reviewer assesses the record and its limitations independently of execution status. Review can reject or qualify a conclusion. A local release records availability and reproducibility limits without certifying safety or external acceptance.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ], "questions": [ { "id": "finding-review-q01", "kind": "authority", "text": "Which qualified reviewer evaluated the contrast and what conflicts of interest were declared?", "answer_data": [ "Reviewer role, expertise scope, independence statement, conflicts" ] }, { "id": "finding-review-q02", "kind": "state", "text": "What review disposition and unresolved issues apply to each reported claim?", "answer_data": [ "Accepted/qualified/rejected/pending disposition, issue IDs, rationale" ] }, { "id": "finding-review-q03", "kind": "interoperability", "text": "Which evidence export profile and pinned mapping preserve links and disclose losses?", "answer_data": [ "Profile version, mapping manifest, omitted fields, validation report" ] }, { "id": "finding-review-q04", "kind": "evidence", "text": "Which released artifacts permit recomputation and which dependencies remain unavailable?", "answer_data": [ "Release manifest, accessible artifacts, restricted dependencies, reproduction status" ] } ], "data_elements": [ { "id": "finding-review-data01", "name": "review_record", "description": "Reviewer dispositions, qualifications and unresolved issues.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] }, { "id": "finding-review-data02", "name": "release_manifest", "description": "Released artifacts, revisions, access conditions and missing dependencies.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] }, { "id": "finding-review-data03", "name": "mapping_profile", "description": "Pinned optional mapping with conformance status and loss report.", "value_kind": "reference", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] } ], "artifacts": [ { "id": "finding-review-artifact", "name": "Review and release manifest", "description": "Versioned evidence for evidence-backed review disposition. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-governance", "name": "Access and retention", "description": "Proposed record obligations for access and retention.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ], "findings": [ { "id": "finding-governance", "name": "Controlled evidence custody", "description": "The proposed study policy separates public conclusions from restricted inputs and outputs. Disclosure, retention and disposal use local authority and rights checks. Dangerous evaluations are represented only by policy-level scope and oversight metadata.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ], "questions": [ { "id": "finding-governance-q01", "kind": "access", "text": "Who may view each bundle, layer, finding and artifact under the evidence access policy?", "answer_data": [ "Scope, role, purpose, grant period and restriction" ] }, { "id": "finding-governance-q02", "kind": "privacy", "text": "Which personal or sensitive content requires minimization or redaction before release?", "answer_data": [ "Classification, redaction decision, restricted original reference" ] }, { "id": "finding-governance-q03", "kind": "retention", "text": "Which retention schedule, preservation holds and disposal rules govern each evidence class?", "answer_data": [ "Schedule authority, expiry, holds, disposal evidence, minimal tombstone" ] }, { "id": "finding-governance-q04", "kind": "security", "text": "What authorization and audit trail govern access changes or restricted evaluation metadata?", "answer_data": [ "Authorizer, scoped change, reason, expiry, audit event" ] } ], "data_elements": [ { "id": "finding-governance-data01", "name": "access_policy", "description": "Role-scoped grants and restriction inheritance.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] }, { "id": "finding-governance-data02", "name": "disclosure_register", "description": "Rights review, redactions and release authorizations.", "value_kind": "collection", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] }, { "id": "finding-governance-data03", "name": "retention_policy", "description": "Retention classes, holds, disposal criteria and safe tombstones.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] } ], "artifacts": [ { "id": "finding-governance-artifact", "name": "Evidence custody policy", "description": "Versioned evidence for controlled evidence custody. Candidate form; exact instance schema remains a hold.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system ID first; otherwise namespace-scoped UUID. Revision and content digest are separate from stable identity; time is metadata.", "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] } ], "inline_only_rationale": null } ] } ] } ] }, "functions": [ { "id": "function-register", "name": "Register study intent", "description": "Proposed, unimplemented local operation: register study intent.", "inputs": [ "study identity, accountable role and protocol draft", "Expected study revision and acting role" ], "outputs": [ "draft revision receipt", "Audit receipt or explicit refusal with reason" ], "preconditions": [ "Acting role is authorized for this study and evidence scope.", "Expected revision matches current revision; referenced evidence is resolvable or explicitly marked unavailable.", "Operation-specific input checks pass; missing critical inputs produce refusal." ], "effects": [ "A new study ID is reserved without creating any execution job.", "Append actor, operation, time and prior/new revision to a policy-controlled audit trail." ], "source_refs": [ "SRC-003", "SRC-005" ] }, { "id": "function-freeze", "name": "Freeze comparison protocol", "description": "Proposed, unimplemented local operation: freeze comparison protocol.", "inputs": [ "arm manifest, corpus and scorer bindings, analysis and failure policies", "Expected study revision and acting role" ], "outputs": [ "frozen protocol receipt or refusal report", "Audit receipt or explicit refusal with reason" ], "preconditions": [ "Acting role is authorized for this study and evidence scope.", "Expected revision matches current revision; referenced evidence is resolvable or explicitly marked unavailable.", "Operation-specific input checks pass; missing critical inputs produce refusal." ], "effects": [ "A local revision becomes immutable; unresolved critical bindings prevent ready-for-execution status.", "Append actor, operation, time and prior/new revision to a policy-controlled audit trail." ], "source_refs": [ "SRC-001", "SRC-003", "SRC-005" ] }, { "id": "function-link", "name": "Link execution evidence", "description": "Proposed, unimplemented local operation: link execution evidence.", "inputs": [ "external run receipts, arm/repeat keys and original observation references", "Expected study revision and acting role" ], "outputs": [ "lineage index revision and mismatch report", "Audit receipt or explicit refusal with reason" ], "preconditions": [ "Acting role is authorized for this study and evidence scope.", "Expected revision matches current revision; referenced evidence is resolvable or explicitly marked unavailable.", "Operation-specific input checks pass; missing critical inputs produce refusal." ], "effects": [ "Verified references are appended; missing or inconsistent evidence remains flagged and no run is launched.", "Append actor, operation, time and prior/new revision to a policy-controlled audit trail." ], "source_refs": [ "SRC-005", "SRC-006" ] }, { "id": "function-check", "name": "Check contrast comparability", "description": "Proposed, unimplemented local operation: check contrast comparability.", "inputs": [ "frozen protocol, arm configuration diffs, scorer receipts and pair keys", "Expected study revision and acting role" ], "outputs": [ "eligible, qualified or ineligible comparison report", "Audit receipt or explicit refusal with reason" ], "preconditions": [ "Acting role is authorized for this study and evidence scope.", "Expected revision matches current revision; referenced evidence is resolvable or explicitly marked unavailable.", "Operation-specific input checks pass; missing critical inputs produce refusal." ], "effects": [ "A local check records corpus/scorer mismatches, unexpected changes, failures and pairing defects without altering source scores.", "Append actor, operation, time and prior/new revision to a policy-controlled audit trail." ], "source_refs": [ "SRC-001", "SRC-002", "SRC-004" ] }, { "id": "function-record", "name": "Record analysis and interpretation", "description": "Proposed, unimplemented local operation: record analysis and interpretation.", "inputs": [ "admitted contrast report, analysis outputs, assumptions and sensitivity evidence", "Expected study revision and acting role" ], "outputs": [ "versioned estimate and claim record or refusal", "Audit receipt or explicit refusal with reason" ], "preconditions": [ "Acting role is authorized for this study and evidence scope.", "Expected revision matches current revision; referenced evidence is resolvable or explicitly marked unavailable.", "Operation-specific input checks pass; missing critical inputs produce refusal." ], "effects": [ "Externally computed estimates are recorded with signed convention and uncertainty; this operation neither chooses a universal test nor executes analysis.", "Append actor, operation, time and prior/new revision to a policy-controlled audit trail." ], "source_refs": [ "SRC-002", "SRC-007", "SRC-008" ] }, { "id": "function-review", "name": "Record review and release decision", "description": "Proposed, unimplemented local operation: record review and release decision.", "inputs": [ "claim record, reviewer scope, release manifest and access policy", "Expected study revision and acting role" ], "outputs": [ "review disposition and local release eligibility record", "Audit receipt or explicit refusal with reason" ], "preconditions": [ "Acting role is authorized for this study and evidence scope.", "Expected revision matches current revision; referenced evidence is resolvable or explicitly marked unavailable.", "Operation-specific input checks pass; missing critical inputs produce refusal." ], "effects": [ "A local decision preserves holds and redactions; it does not publish externally, approve deployment or erase evidence.", "Append actor, operation, time and prior/new revision to a policy-controlled audit trail." ], "source_refs": [ "SRC-005", "SRC-006", "SRC-008" ] } ], "composition": [ { "target": "WM-AI-003", "relation": "ALIGN", "purpose": "Candidate conceptual specialization from registry parent; executable inheritance remains unverified.", "required": false, "source_refs": [ "SRC-001", "SRC-003" ] }, { "target": "WM-AI-009", "relation": "REFERENCE", "purpose": "Candidate frozen evaluation corpus binding; required case identity is a study obligation, not a forced dependency on this package.", "required": false, "source_refs": [ "SRC-005", "SRC-006" ] }, { "target": "WM-AI-011", "relation": "REFERENCE", "purpose": "Candidate pinned scorer and harness definition; no scorer code is mastered here.", "required": false, "source_refs": [ "SRC-002", "SRC-005" ] }, { "target": "WM-AI-012", "relation": "REFERENCE", "purpose": "Candidate external result evidence for each arm or contrast.", "required": false, "source_refs": [ "SRC-002", "SRC-005" ] }, { "target": "WM-AI-004", "relation": "REFERENCE", "purpose": "Optional external inference run receipts indexed by arm and repeat.", "required": false, "source_refs": [ "SRC-005", "SRC-008" ] }, { "target": "WM-AI-006", "relation": "REFERENCE", "purpose": "Optional training run references for interventions involving retraining.", "required": false, "source_refs": [ "SRC-005", "SRC-008" ] }, { "target": "WM-AI-014", "relation": "REFERENCE", "purpose": "Optional label set revision when the evaluation uses ground truth.", "required": false, "source_refs": [ "SRC-005", "SRC-006" ] }, { "target": "https://www.w3.org/TR/prov-o/", "relation": "ALIGN", "purpose": "Conceptual provenance mapping only; no implemented conformance claim.", "required": false, "source_refs": [ "SRC-005" ] }, { "target": "https://www.researchobject.org/ro-crate/specification/1.2/introduction.html", "relation": "ALIGN", "purpose": "Optional evidence packaging profile; selected 1.2 must be pinned and mapped before use.", "required": false, "source_refs": [ "SRC-006" ] } ], "serviceLayers": { "dimension": { "owner_package_requirements": [ "Accountable study steward role and delegated authority record.", "Owner-controlled namespace, storage binding and revision policy.", "Evidence access, retention and review policies with named responsible roles.", "Pinned corpus, scorer and neighbor bindings before operational use." ], "namespace_guidance": "Use an owner-controlled URI namespace plus stable study and artifact IDs. Do not place dates or commercial owner names in canonical local node IDs.", "registry_links": [ "vr.wm-ai-013", "WM-AI-003 conceptual parent; binding unverified" ] }, "canon_and_patch": { "canonicalization_rules": [ "Separate study identity, protocol revision, execution attempt, evidence digest and interpretation revision.", "This research package remains reviewable-draft; canonical promotion requires independent review and closure of holds." ], "patch_rules": [ "Use expected-revision checks and append provenance for changes.", "A corpus, scorer, intervention or analysis-rule change creates a new protocol/analysis revision; preserve the old comparison and mark outcome-informed changes." ], "compatibility_rules": [ "Renaming a display label can preserve IDs; changing metric direction, pairing unit or intervention meaning requires explicit migration.", "Mappings must report dropped fields and unresolved references; similar labels do not establish equivalent semantics." ] }, "artifact_rules": { "identity_priority": [ "Authoritative master-system identifier in its namespace.", "Stable owner-scoped identifier or UUID when no master identifier exists.", "Content hash for integrity and deduplication, not a substitute for logical identity." ], "timestamp_rule": "Record known times in RFC 3339 with seconds and an explicit UTC offset or Z; preserve clock uncertainty and do not invent unknown times.", "serial_naming_rule": "Use stable study/artifact IDs and separate immutable revision IDs; filenames and timestamps are metadata, not primary identity.", "integrity_rule": "Record hash algorithm, digest, byte scope, media type and provenance. A matching digest proves byte consistency, not correctness or authenticity." }, "policies": [ "All services are proposed local record operations, not executable training or deployment capabilities.", "Restrict harmful evaluation details to policy-level purpose, authorization, oversight and disclosure decisions.", "Respect source and dataset rights, personal-data minimization and release restrictions.", "Preserve failed attempts and negative findings under the applicable retention policy; reproducibility does not mandate indefinite storage." ], "crud": { "read": [ "Authorize at every scope; show missing or restricted evidence rather than filling gaps with invented facts." ], "create": [ "Create a study draft with stable identity, responsible role and objective. Empty result slots are allowed before execution, not as fabricated evidence." ], "update": [ "Use revision preconditions, scoped authority and append-only decision provenance subject to retention policy. Never overwrite original observations with retries." ], "delete": [ "Retire or supersede records first; deletion requires applicable retention and hold checks, authority and an audit receipt.", "When lawful disposal applies, remove restricted payloads and retain only permitted minimal tombstones; historical references do not override erasure obligations." ] }, "roles": [ { "name": "Study steward", "responsibilities": [ "Own purpose, scope, protocol revisions and lifecycle decisions." ] }, { "name": "Experiment operator", "responsibilities": [ "Supply external run and configuration evidence without changing the frozen protocol." ] }, { "name": "Analysis reviewer", "responsibilities": [ "Assess pairing, assumptions, uncertainty and bounded claims; declare independence limits." ] }, { "name": "Evidence custodian", "responsibilities": [ "Enforce access, integrity, retention and disclosure rules." ] }, { "name": "Release reviewer", "responsibilities": [ "Review rights and unresolved holds before recording release eligibility." ] } ], "access": { "default_rule": "Deny access unless owner policy grants a role and purpose at the relevant scope; public summaries do not imply public raw data.", "scopes": [ "bundle", "layer", "finding", "artifact" ], "exceptions": [ "Time-limited approved review access must record purpose, authority and expiry.", "Redacted derivatives have separate IDs and preserve links to restricted originals without exposing their payloads." ], "audit_requirements": [ "Record actor, purpose, scope, operation, outcome and RFC 3339 time for access changes and mutations.", "Review inherited grants when a new artifact or classification is introduced." ] }, "agents_bootstrap": { "filename": "AGENTS.md", "required_fields": [ "Name", "Type", "Specification URL", "Storage type URL", "Interface URL", "Processes URL" ], "read_order": [ "Read local AGENTS.md and owner policy.", "Read spec.yaml, research holds and pinned dependency contracts.", "Resolve actual case/scorer/run identities before populating findings." ] } }, "coverage": { "claim": "Source-grounded proposed structure for one controlled ablation study record, assessed in a separate frozen local no-tools self-audit. It preserves common-corpus and common-scorer contrasts, evidence lineage and qualified inference. This is a noncanonical reviewable draft; no experiment, executable conformance or independent external review is claimed.", "confidence": "medium", "checklist": [ { "dimension": "identity", "status": "covered", "notes": "Stable study identity with separate protocol, run and artifact revisions." }, { "dimension": "lifecycle", "status": "covered", "notes": "Design, execution, analysis, review, closure and withdrawal decisions." }, { "dimension": "relationships", "status": "covered", "notes": "Candidate external corpus, labels, scorer, runs and results remain separately mastered." }, { "dimension": "temporal", "status": "covered", "notes": "Freeze, outcome inspection, execution and amendment times distinguish prespecification." }, { "dimension": "provenance", "status": "covered", "notes": "Configuration and evidence lineage preserve source and revision links." }, { "dimension": "ownership", "status": "covered", "notes": "Role-based stewardship, custody and delegated authority." }, { "dimension": "validation", "status": "gap", "notes": "Research structure validates; nested instance schemas and executable fixtures are not implemented." }, { "dimension": "access", "status": "covered", "notes": "Scoped grants, redaction and restricted original evidence." }, { "dimension": "interoperability", "status": "gap", "notes": "PROV-O and RO-Crate are conceptual mappings pending pinned conformance tests." }, { "dimension": "properties", "status": "covered", "notes": "Direct properties are study class, hypothesis, status, intervention phase and protocol revision." }, { "dimension": "recognition", "status": "covered", "notes": "Recognize an instance by an identified study with controlled arm contrast intent and evidence lineage." }, { "dimension": "capabilities", "status": "covered", "notes": "Six proposed local functions record and check evidence; actual execution is delegated." }, { "dimension": "physical geometry", "status": "not-applicable", "notes": "The abstract study has no intrinsic mass or geometry; execution resource measurements belong to referenced runs." }, { "dimension": "statistical inference", "status": "gap", "notes": "Metric-specific tests, dependence handling and equivalence margins need qualified profile review." }, { "dimension": "source verification", "status": "gap", "notes": "Web-service evidence reviewed; direct HTTP unattempted under sandbox restriction, immutable versions not fully verified." }, { "dimension": "independent review", "status": "gap", "notes": "Codex only; external providers waived. Separate local no-tools self-audit is still required." } ], "known_omissions": [ "Independent external provider review is absent.", "Direct HTTP status and immutable source versions remain unverified.", "Nested instance schemas, profile-specific validators and acceptance fixtures remain unimplemented.", "Statistical methods for particular metrics, clusters, adaptive designs and resource-constrained studies need qualified selection.", "Neighbor model contracts and machine-executable mappings remain unpinned." ], "conflicts": [], "regional_assumptions": [ "No jurisdiction or sector is assumed. Adopters must supply applicable evidence rights, privacy, ethics and retention rules.", "English conceptual draft; no localization or legal compliance claim." ], "adversarial_checks": [ "A changed scorer or case cohort must not be admitted as an unchanged controlled comparison.", "A retraining intervention must not be labelled as inference-only disabling.", "Same seed labels do not establish independent repeats or perfect pairing.", "Failed runs and missing pairs cannot disappear from the denominator without explanation.", "A non-significant difference does not establish equivalence or absence of contribution.", "No source or provenance vocabulary establishes causation or runtime conformance." ] }, "researchAdjudication": { "providerMode": "single-provider-waiver", "activeProviders": [ "codex" ], "waivedProviders": [ "claude", "grok" ], "providerPolicy": { "contract_version": "1.0.0", "mode": "single-provider-waiver", "effective_at": "2026-09-06T00:00:00Z", "scope": "Canonical single-stream subject-model research after the six-workstream consolidation", "active_providers": [ "codex" ], "waived_providers": [ { "provider": "claude", "authorized_by": "repository owner", "authorized_at": "2026-09-06T00:00:00Z", "reason": "Claude produced no result on prior 1800-second and 900-second attempts and again timed out on bounded 600-second Sonnet and 300-second Haiku passes. The owner prioritized completion over provider availability." }, { "provider": "grok", "authorized_by": "repository owner", "authorized_at": "2026-09-06T00:00:00Z", "reason": "The repository owner authorized completion without Grok when Grok is unavailable, slow or schema-invalid. Grok may still be attempted as a bounded supplemental reviewer, but its failure never blocks a valid Claude plus no-tools result." } ], "review_rule": "Codex may complete source-grounded fallback research after bounded Claude and Grok attempts fail. It requires a separate no-tools adversarial audit and remains reviewable-draft with a visible absence-of-external-review hold.", "supplemental_provider_attempts": [ { "provider": "claude", "required": false, "maximum_attempts": 1, "failure_policy": "record-and-continue", "admission_rule": "Use only a locally schema-valid result whose sources and boundaries survive adjudication." }, { "provider": "grok", "required": false, "maximum_attempts": 1, "failure_policy": "record-and-continue", "admission_rule": "Use only a locally schema-valid result whose sources and boundaries survive adjudication." } ] }, "boundaryDecision": { "entry_kind": "entity", "status": "accepted", "rationale": "One persistent study record retains identity across protocol, evidence and interpretation revisions. Referenced runs, datasets, labels, scorers and benchmark results remain externally mastered. The registry standalone-mm value classifies the record plane, not subject kind; parent alignment does not establish executable inheritance." }, "decisions": [ { "concept": "Study identity and root", "disposition": "accepted", "rationale": "Stable study identity and revision questions justify an entity. Neither a reusable recipe nor an execution run can substitute for this root." }, { "concept": "Registry and legacy reconciliation", "disposition": "qualified", "rationale": "The registry parent is conceptual alignment only, with no matching relationship row. Empty legacy fields and an absent supplement provide no further specification authority; historical registry provenance is not this run's review." }, { "concept": "Protocol freeze and amendments", "disposition": "accepted", "rationale": "Freeze and outcome-inspection times preserve the distinction between prespecified and outcome-informed analysis, while revisions and lifecycle events remain traceable." }, { "concept": "Intervention and nuisance controls", "disposition": "accepted with limits", "rationale": "Training-time and inference-time interventions remain distinct. Configuration differences, resource policy and background settings bound attribution; isolated ablations do not identify every interaction." }, { "concept": "Common corpus and label binding", "disposition": "accepted", "rationale": "Both arms preserve evaluation case identity and membership. Label applicability is conditional, while drift, omissions and corrections require explicit disposition instead of silently redefining the cohort." }, { "concept": "Shared scorer contract", "disposition": "accepted", "rationale": "Scorer identity, version, rubric, metric direction and aggregation are pinned across arms. Stochastic scoring has a separate policy, and scorer changes require both arms to be rescored under a new revision." }, { "concept": "Execution evidence and failed attempts", "disposition": "accepted", "rationale": "External run masters and pair keys preserve original attempts, missingness and denominators. Retries cannot overwrite failures, and seed labels alone do not establish independent replication." }, { "concept": "Delta and uncertainty", "disposition": "qualified", "rationale": "The with-minus-without convention is explicit and depends on metric direction. Sampling units, clustering and assumptions are required questions; dataset-level statistical guidance cannot mandate a within-corpus test." }, { "concept": "Conclusion and equivalence", "disposition": "qualified", "rationale": "The draft separates observed differences, bounded explanations and adoption decisions. Multiplicity and selection history remain visible; a non-significant contrast is not evidence of equivalence." }, { "concept": "Review and local functions", "disposition": "accepted as proposed", "rationale": "All six functions are explicitly unimplemented local record operations with role and revision checks and refusal results. Review disposition does not execute scoring, publish externally or authorize deployment." }, { "concept": "Evidence identity and custody", "disposition": "accepted with policy dependency", "rationale": "Master IDs precede fallback IDs and hashes, while revisions and times remain separate metadata. Restricted payloads and redacted derivatives obey scoped access, retention and permitted disposal rather than perpetual storage." }, { "concept": "Instance schemas and mappings", "disposition": "deferred", "rationale": "Optional candidate fields and narrative preconditions do not constitute executable instance validation. Nested schemas, pinned neighbor contracts, mapping tests and adoption fixtures remain explicit conformance holds." }, { "concept": "Source support and access limits", "disposition": "limited", "rationale": "Selected web-service evidence grounds methodological and provenance principles, not externally standardized field names. One scientific source was abstract-only; direct HTTP checks were unattempted and immutable version verification remains open." }, { "concept": "Independent review and confidence", "disposition": "waived and held", "rationale": "The no-tools phase is a separate local Codex self-audit of frozen evidence, not an independent provider. No waived-provider additions are admitted, confidence remains medium and canonical promotion is unavailable." } ], "publicationHolds": [ "Independent external review is absent under the owner-authorized single-provider waiver. Claude and Grok were skipped with zero attempts. The separate local Codex no-tools self-audit is not an independent second-provider review.", "Live source and immutable-version verification remains incomplete. Eight source URLs supplied selected web-service text, including one abstract-only source. Direct HTTP checks were not attempted under the owner-reported sandbox restriction; zero HTTP 200 responses were measured and no direct status is known. The coordinator checker is prepared.", "Qualified statistical profile review is required before operational use: choose appropriate sampling units, dependence handling, metric-specific uncertainty, multiplicity policy, equivalence margins and causal/generalization limits. No actual experiment or empirical effect is claimed.", "Nested instance schemas, executable comparison validators, pinned neighbor bindings, PROV-O and RO-Crate mappings and adversarial acceptance fixtures remain incomplete. Structural research validation is not runtime conformance.", "Adoption-specific evidence rights, privacy, retention, disclosure, ethics and restricted-evaluation policy require accountable review before operational use. The package provides no jurisdictional or safety certification.", "Independent external review was explicitly waived by the repository owner; this codex-only result remains a reviewable draft." ], "deferredResearch": [ "Complete direct source checks and selected-version verification, review the full statistical paper if a profile needs its methods, and restore independent external review before canonical promotion.", "Develop nested instance schemas and fixtures for scorer drift, unmatched pairs, failed retries, label revisions, budget imbalance and outcome-informed protocol amendments.", "Validate metric-specific analysis profiles with dependent observations, stochastic scorers, multiple contrasts and non-estimable uncertainty, and pin external model contracts and evidence export mappings." ] }, "statistics": { "sources": 8, "bundles": 6, "layers": 12, "findings": 12, "questions": 48, "artifacts": 12, "functions": 6 } }