# Vercy AI instruction - YAML 1.2 (JSON-compatible) { "vercy": "1.0-draft", "publication": { "status": "published", "adjudicationStatus": "reviewable-draft", "publishableCanonical": false, "generatedAt": "2026-09-06T12:38:16Z", "synthesisSha256": "d7b6ff0135f7213057b464e76eeaecf2c6b4b68757ce79b12f889a7d37aad9cc", "providerMode": "single-provider-waiver", "providers": [ "Codex" ], "waivedProviders": [ "Claude", "Grok" ] }, "metaModel": { "id": "WM-DAT-005", "registryId": "vr.wm-dat-005", "name": "Data Pipeline", "version": "0.3.0-research.1", "previousVersions": [], "entryKind": "aggregate", "family": "World Models", "category": "Information and virtual systems", "industry": [ "Cross-industry" ], "domain": [ "INF.DAT.PIP" ], "tags": [ "data", "pipeline", "inf.dat.pip" ], "status": "published" }, "canonicalUrl": "https://ver.cy/models/wm-dat-005-data-pipeline/", "sourceUrl": "https://github.com/ver-cy/world-models/tree/feat/mega-model-registry/publications/wm-dat-005-data-pipeline", "model": { "registry_id": "vr.wm-dat-005", "model_id": "WM-DAT-005", "name": "Data Pipeline", "entry_kind": "aggregate", "purpose": "Represent one governed reusable data pipeline definition whose topology, interfaces, logic references, policies and controls can be understood and projected without absorbing runtime, dataset or lineage-event masters.", "scope_statement": "Owns pipeline identity, version and lifecycle; purpose, domain and ownership; component graph, ports and dependency semantics; dataset and data-contract bindings; logic, artifact, configuration, environment and secret references; trigger, schedule and execution policies; intended lineage, validation, quality, observability, security, privacy, resilience and interoperability projections. Workflow, run, task attempt, dataset, data product, lineage event, deployment, scheduler, software, secret, telemetry, incident, evidence, audit and records masters remain external.", "in_scope": [ "Pipeline identity, version, ownership, purpose, topology, components, ports, edges, dependencies, contracts, logic references, parameters, configuration and execution policies", "Intended lineage, quality, observability, security, privacy, resilience, lifecycle, releases, runtime bindings, run references, catalog views, audit and projections" ], "out_of_scope": [ "Owning workflow, run, task attempt, dataset, data product, lineage event, deployment, scheduler, software, credential, telemetry, incident, evidence, audit or records masters", "Treating intended topology, a completed run or an emitted lineage event as proof of quality, freshness, publication, consumption, compliance or exactly-once behavior", "Autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition" ], "boundary_notes": [ { "neighbor": "WM-ACT-003 Process / Workflow", "distinction": "The unified parent signal supplies a general workflow boundary but grants no inheritance or ownership. The data pipeline specializes data movement and transformation while preserving its own versioned definition.", "source_refs": [ "SRC-002", "SRC-003", "SRC-004" ] }, { "neighbor": "WM-ACT-053 Data Processing Job / Pipeline Run", "distinction": "The run master owns a time-bounded execution, attempts, input and output instances, status and runtime events. The pipeline owns reusable intended structure and policy.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-012" ] }, { "neighbor": "WM-DAT-006 Data Lineage", "distinction": "The lineage master owns cross-system provenance observations and their evidence. The pipeline stores expected derivations and non-owning lineage references.", "source_refs": [ "SRC-001", "SRC-005" ] }, { "neighbor": "Dataset, data product, contract, deployment, scheduler, software, secret and telemetry", "distinction": "These masters own data semantics, product accountability, interface commitments, runtime state, executable artifacts, credentials and observations. The pipeline binds version-qualified references only.", "source_refs": [ "SRC-001", "SRC-002", "SRC-006", "SRC-007", "SRC-009", "SRC-011", "SRC-013" ] }, { "neighbor": "BPMN, CWL, Airflow, OpenLineage, PROV, DCAT, DQV, SHACL, JSON Schema, CloudEvents, OpenTelemetry, Kubernetes and OCI", "distinction": "These standards and platforms have distinct scopes. Every mapping is version-pinned, profile-qualified and loss-declaring, never presumed universal or lossless.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013" ] } ] }, "sources": [ { "id": "SRC-001", "title": "OpenLineage Object Model", "organization": "OpenLineage", "url": "https://openlineage.io/docs/spec/object-model/", "version_or_date": "Specification 1.53.0 current at access", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Separates Job and Dataset identities from RunEvent observations and defines extensible facets." }, { "id": "SRC-002", "title": "DAGs", "organization": "Apache Software Foundation", "url": "https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/dags.html", "version_or_date": "Apache Airflow 3.3.0 documentation current at access", "source_type": "first-party-doc", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines reusable DAG topology, schedule, tasks and dependencies separately from DAG runs and task instances." }, { "id": "SRC-003", "title": "Common Workflow Language Workflow Description", "organization": "Common Workflow Language Project", "url": "https://www.commonwl.org/v1.2/Workflow.html", "version_or_date": "CWL v1.2.1", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines portable workflow inputs, outputs, steps, requirements, hints and execution setup." }, { "id": "SRC-004", "title": "Business Process Model and Notation", "organization": "Object Management Group", "url": "https://www.omg.org/spec/BPMN/2.0.2/", "version_or_date": "BPMN 2.0.2 January 2014", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines business process and control-flow notation used as a bounded projection." }, { "id": "SRC-005", "title": "PROV-O The PROV Ontology", "organization": "World Wide Web Consortium", "url": "https://www.w3.org/TR/prov-o/", "version_or_date": "W3C Recommendation 30 April 2013", "source_type": "ontology", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines entity, activity, agent, attribution, derivation, revision and temporal provenance." }, { "id": "SRC-006", "title": "Data Catalog Vocabulary DCAT Version 3", "organization": "World Wide Web Consortium", "url": "https://www.w3.org/TR/vocab-dcat-3/", "version_or_date": "W3C Recommendation 22 August 2024", "source_type": "ontology", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines dataset, data service, distribution, catalog and relationship projections." }, { "id": "SRC-007", "title": "Data on the Web Best Practices Data Quality Vocabulary", "organization": "World Wide Web Consortium", "url": "https://www.w3.org/TR/vocab-dqv/", "version_or_date": "W3C Note 15 December 2016", "source_type": "ontology", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines quality measurement, annotation and policy concepts." }, { "id": "SRC-008", "title": "Shapes Constraint Language SHACL", "organization": "World Wide Web Consortium", "url": "https://www.w3.org/TR/shacl/", "version_or_date": "W3C Recommendation 20 July 2017", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines RDF graph validation shapes and results." }, { "id": "SRC-009", "title": "JSON Schema Draft 2020-12", "organization": "JSON Schema Project", "url": "https://json-schema.org/draft/2020-12", "version_or_date": "Published 16 June 2022", "source_type": "schema", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines schema vocabularies, bundling and validation for JSON instances." }, { "id": "SRC-010", "title": "CloudEvents Specification", "organization": "Cloud Native Computing Foundation", "url": "https://github.com/cloudevents/spec", "version_or_date": "CloudEvents 1.0.2 released 6 February 2022", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines a portable event-envelope projection for trigger and lifecycle events." }, { "id": "SRC-011", "title": "OpenTelemetry Semantic Conventions", "organization": "OpenTelemetry", "url": "https://opentelemetry.io/docs/specs/semconv/", "version_or_date": "Semantic conventions 1.44.0 current at access", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines versioned attribute naming for traces, metrics, logs and events with mixed stability." }, { "id": "SRC-012", "title": "Jobs", "organization": "Kubernetes", "url": "https://kubernetes.io/docs/concepts/workloads/controllers/job/", "version_or_date": "Kubernetes documentation current at access", "source_type": "first-party-doc", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines a finite workload execution, completion, retry and parallelism profile." }, { "id": "SRC-013", "title": "OCI Image Format Specification", "organization": "Open Container Initiative", "url": "https://github.com/opencontainers/image-spec", "version_or_date": "OCI Image Specification 1.1.1", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines content-addressed runtime images, manifests and configuration artifacts." }, { "id": "SRC-014", "title": "Secure Software Development Framework Version 1.1", "organization": "National Institute of Standards and Technology", "url": "https://csrc.nist.gov/pubs/sp/800/218/final", "version_or_date": "NIST SP 800-218 February 2022", "source_type": "public-authority", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Supports governed software changes, provenance, protection and release practices." }, { "id": "SRC-015", "title": "Time Ontology in OWL", "organization": "World Wide Web Consortium", "url": "https://www.w3.org/TR/owl-time/", "version_or_date": "W3C Recommendation 19 October 2017", "source_type": "ontology", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines instants, intervals, durations, temporal reference systems and relations." }, { "id": "SRC-016", "title": "Date and Time on the Internet Timestamps", "organization": "Internet Engineering Task Force", "url": "https://www.rfc-editor.org/info/rfc3339/", "version_or_date": "RFC 3339 July 2002", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-09-06T12:40:00Z", "relevance": "Defines interoperable timestamps with seconds and explicit UTC relationship." } ], "structure": { "bundles": [ { "id": "pipeline-identity-scope-ownership-and-definition", "name": "Pipeline identity, scope, ownership and definition", "description": "Groups governed pipeline context for pipeline identity, scope, ownership and definition.", "rationale": "The reusable pipeline definition retains a stable identity independently of deployment and execution.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-006", "SRC-014" ], "layers": [ { "id": "pipeline-root-version-state-and-namespace", "name": "Pipeline root, version, state and namespace", "description": "Groups governed pipeline context for pipeline root, version, state and namespace.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-014" ], "findings": [ { "id": "pipeline-identity-type-version-current-head-and-revision", "name": "Pipeline identity, type, version, current head and revision", "description": "Records pipeline identity, type, version, current head and revision as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-014" ], "questions": [ { "id": "pipeline-identity-type-version-current-head-and-revision-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish pipeline identity, type, version, current head and revision?", "kind": "identity", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "pipeline-identity-type-version-current-head-and-revision-q02", "text": "Who declared, owns, approves, operates, observes or may rely on pipeline identity, type, version, current head and revision, under which authority and limits?", "kind": "event", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "pipeline-identity-type-version-current-head-and-revision-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify pipeline identity, type, version, current head and revision?", "kind": "process", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "pipeline-identity-type-version-current-head-and-revision-data", "name": "Pipeline identity, type, version, current head and revision data", "description": "Typed pipeline-definition data for pipeline identity, type, version, current head and revision, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-014" ] } ], "artifacts": [ { "id": "pipeline-identity-type-version-current-head-and-revision-record", "name": "Pipeline identity, type, version, current head and revision record", "description": "Immutable or successor-versioned pipeline evidence for pipeline identity, type, version, current head and revision.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for pipeline-identity-type-version-current-head-and-revision; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-014" ] } ], "inline_only_rationale": null }, { "id": "namespace-domain-data-product-purpose-criticality-and-classification", "name": "Namespace, domain, data product, purpose, criticality and classification", "description": "Records namespace, domain, data product, purpose, criticality and classification as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-014" ], "questions": [ { "id": "namespace-domain-data-product-purpose-criticality-and-classification-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish namespace, domain, data product, purpose, criticality and classification?", "kind": "classification", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "namespace-domain-data-product-purpose-criticality-and-classification-q02", "text": "Who declared, owns, approves, operates, observes or may rely on namespace, domain, data product, purpose, criticality and classification, under which authority and limits?", "kind": "temporal", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "namespace-domain-data-product-purpose-criticality-and-classification-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify namespace, domain, data product, purpose, criticality and classification?", "kind": "validation", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "namespace-domain-data-product-purpose-criticality-and-classification-data", "name": "Namespace, domain, data product, purpose, criticality and classification data", "description": "Typed pipeline-definition data for namespace, domain, data product, purpose, criticality and classification, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-014" ] } ], "artifacts": [ { "id": "namespace-domain-data-product-purpose-criticality-and-classification-record", "name": "Namespace, domain, data product, purpose, criticality and classification record", "description": "Immutable or successor-versioned pipeline evidence for namespace, domain, data product, purpose, criticality and classification.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for namespace-domain-data-product-purpose-criticality-and-classification; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-014" ] } ], "inline_only_rationale": null } ] }, { "id": "ownership-stakeholders-and-definition-boundaries", "name": "Ownership, stakeholders and definition boundaries", "description": "Groups governed pipeline context for ownership, stakeholders and definition boundaries.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-006", "SRC-014" ], "findings": [ { "id": "owner-steward-author-operator-approver-consumer-and-contact", "name": "Owner, steward, author, operator, approver, consumer and contact", "description": "Records owner, steward, author, operator, approver, consumer and contact as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-006", "SRC-014" ], "questions": [ { "id": "owner-steward-author-operator-approver-consumer-and-contact-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish owner, steward, author, operator, approver, consumer and contact?", "kind": "ownership", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "owner-steward-author-operator-approver-consumer-and-contact-q02", "text": "Who declared, owns, approves, operates, observes or may rely on owner, steward, author, operator, approver, consumer and contact, under which authority and limits?", "kind": "composition", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "owner-steward-author-operator-approver-consumer-and-contact-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify owner, steward, author, operator, approver, consumer and contact?", "kind": "privacy", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "owner-steward-author-operator-approver-consumer-and-contact-data", "name": "Owner, steward, author, operator, approver, consumer and contact data", "description": "Typed pipeline-definition data for owner, steward, author, operator, approver, consumer and contact, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-006", "SRC-014" ] } ], "artifacts": [ { "id": "owner-steward-author-operator-approver-consumer-and-contact-record", "name": "Owner, steward, author, operator, approver, consumer and contact record", "description": "Immutable or successor-versioned pipeline evidence for owner, steward, author, operator, approver, consumer and contact.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for owner-steward-author-operator-approver-consumer-and-contact; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-006", "SRC-014" ] } ], "inline_only_rationale": null }, { "id": "workflow-release-deployment-scheduler-run-and-task-boundary", "name": "Workflow, release, deployment, scheduler, run and task boundary", "description": "Records workflow, release, deployment, scheduler, run and task boundary as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-006", "SRC-014" ], "questions": [ { "id": "workflow-release-deployment-scheduler-run-and-task-boundary-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish workflow, release, deployment, scheduler, run and task boundary?", "kind": "relationship", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "workflow-release-deployment-scheduler-run-and-task-boundary-q02", "text": "Who declared, owns, approves, operates, observes or may rely on workflow, release, deployment, scheduler, run and task boundary, under which authority and limits?", "kind": "evidence", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "workflow-release-deployment-scheduler-run-and-task-boundary-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify workflow, release, deployment, scheduler, run and task boundary?", "kind": "lifecycle", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "workflow-release-deployment-scheduler-run-and-task-boundary-data", "name": "Workflow, release, deployment, scheduler, run and task boundary data", "description": "Typed pipeline-definition data for workflow, release, deployment, scheduler, run and task boundary, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-006", "SRC-014" ] } ], "artifacts": [ { "id": "workflow-release-deployment-scheduler-run-and-task-boundary-record", "name": "Workflow, release, deployment, scheduler, run and task boundary record", "description": "Immutable or successor-versioned pipeline evidence for workflow, release, deployment, scheduler, run and task boundary.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for workflow-release-deployment-scheduler-run-and-task-boundary; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-006", "SRC-014" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "topology-components-ports-and-data-contracts", "name": "Topology, components, ports and data contracts", "description": "Groups governed pipeline context for topology, components, ports and data contracts.", "rationale": "Directed topology and declared interfaces make composition explicit without executing it.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ], "layers": [ { "id": "nodes-edges-dependencies-and-control-flow", "name": "Nodes, edges, dependencies and control flow", "description": "Groups governed pipeline context for nodes, edges, dependencies and control flow.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005" ], "findings": [ { "id": "source-transform-sink-gateway-subpipeline-and-component-node", "name": "Source, transform, sink, gateway, subpipeline and component node", "description": "Records source, transform, sink, gateway, subpipeline and component node as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005" ], "questions": [ { "id": "source-transform-sink-gateway-subpipeline-and-component-node-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish source, transform, sink, gateway, subpipeline and component node?", "kind": "composition", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "source-transform-sink-gateway-subpipeline-and-component-node-q02", "text": "Who declared, owns, approves, operates, observes or may rely on source, transform, sink, gateway, subpipeline and component node, under which authority and limits?", "kind": "ownership", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "source-transform-sink-gateway-subpipeline-and-component-node-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify source, transform, sink, gateway, subpipeline and component node?", "kind": "quality", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "source-transform-sink-gateway-subpipeline-and-component-node-data", "name": "Source, transform, sink, gateway, subpipeline and component node data", "description": "Typed pipeline-definition data for source, transform, sink, gateway, subpipeline and component node, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005" ] } ], "artifacts": [ { "id": "source-transform-sink-gateway-subpipeline-and-component-node-record", "name": "Source, transform, sink, gateway, subpipeline and component node record", "description": "Immutable or successor-versioned pipeline evidence for source, transform, sink, gateway, subpipeline and component node.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for source-transform-sink-gateway-subpipeline-and-component-node; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005" ] } ], "inline_only_rationale": null }, { "id": "data-control-dependency-condition-order-cycle-fan-in-and-fan-out-edge", "name": "Data, control, dependency, condition, order, cycle, fan-in and fan-out edge", "description": "Records data, control, dependency, condition, order, cycle, fan-in and fan-out edge as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005" ], "questions": [ { "id": "data-control-dependency-condition-order-cycle-fan-in-and-fan-out-edge-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish data, control, dependency, condition, order, cycle, fan-in and fan-out edge?", "kind": "relationship", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "data-control-dependency-condition-order-cycle-fan-in-and-fan-out-edge-q02", "text": "Who declared, owns, approves, operates, observes or may rely on data, control, dependency, condition, order, cycle, fan-in and fan-out edge, under which authority and limits?", "kind": "measurement", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "data-control-dependency-condition-order-cycle-fan-in-and-fan-out-edge-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify data, control, dependency, condition, order, cycle, fan-in and fan-out edge?", "kind": "security", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "data-control-dependency-condition-order-cycle-fan-in-and-fan-out-edge-data", "name": "Data, control, dependency, condition, order, cycle, fan-in and fan-out edge data", "description": "Typed pipeline-definition data for data, control, dependency, condition, order, cycle, fan-in and fan-out edge, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005" ] } ], "artifacts": [ { "id": "data-control-dependency-condition-order-cycle-fan-in-and-fan-out-edge-record", "name": "Data, control, dependency, condition, order, cycle, fan-in and fan-out edge record", "description": "Immutable or successor-versioned pipeline evidence for data, control, dependency, condition, order, cycle, fan-in and fan-out edge.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for data-control-dependency-condition-order-cycle-fan-in-and-fan-out-edge; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005" ] } ], "inline_only_rationale": null } ] }, { "id": "ports-datasets-streams-schemas-and-contracts", "name": "Ports, datasets, streams, schemas and contracts", "description": "Groups governed pipeline context for ports, datasets, streams, schemas and contracts.", "source_refs": [ "SRC-001", "SRC-003", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ], "findings": [ { "id": "input-output-port-dataset-stream-table-topic-object-and-format", "name": "Input or output port, dataset, stream, table, topic, object and format", "description": "Records input or output port, dataset, stream, table, topic, object and format as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-003", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ], "questions": [ { "id": "input-output-port-dataset-stream-table-topic-object-and-format-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish input or output port, dataset, stream, table, topic, object and format?", "kind": "interoperability", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "input-output-port-dataset-stream-table-topic-object-and-format-q02", "text": "Who declared, owns, approves, operates, observes or may rely on input or output port, dataset, stream, table, topic, object and format, under which authority and limits?", "kind": "exception", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "input-output-port-dataset-stream-table-topic-object-and-format-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify input or output port, dataset, stream, table, topic, object and format?", "kind": "retention", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "input-output-port-dataset-stream-table-topic-object-and-format-data", "name": "Input or output port, dataset, stream, table, topic, object and format data", "description": "Typed pipeline-definition data for input or output port, dataset, stream, table, topic, object and format, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-003", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ] } ], "artifacts": [ { "id": "input-output-port-dataset-stream-table-topic-object-and-format-record", "name": "Input or output port, dataset, stream, table, topic, object and format record", "description": "Immutable or successor-versioned pipeline evidence for input or output port, dataset, stream, table, topic, object and format.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for input-output-port-dataset-stream-table-topic-object-and-format; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-003", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ] } ], "inline_only_rationale": null }, { "id": "data-contract-key-type-nullability-partition-order-compatibility-and-sla", "name": "Data contract, key, type, nullability, partition, order, compatibility and SLA", "description": "Records data contract, key, type, nullability, partition, order, compatibility and sla as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-003", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ], "questions": [ { "id": "data-contract-key-type-nullability-partition-order-compatibility-and-sla-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish data contract, key, type, nullability, partition, order, compatibility and sla?", "kind": "constraint", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "data-contract-key-type-nullability-partition-order-compatibility-and-sla-q02", "text": "Who declared, owns, approves, operates, observes or may rely on data contract, key, type, nullability, partition, order, compatibility and sla, under which authority and limits?", "kind": "provenance", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "data-contract-key-type-nullability-partition-order-compatibility-and-sla-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify data contract, key, type, nullability, partition, order, compatibility and sla?", "kind": "interoperability", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "data-contract-key-type-nullability-partition-order-compatibility-and-sla-data", "name": "Data contract, key, type, nullability, partition, order, compatibility and SLA data", "description": "Typed pipeline-definition data for data contract, key, type, nullability, partition, order, compatibility and sla, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-003", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ] } ], "artifacts": [ { "id": "data-contract-key-type-nullability-partition-order-compatibility-and-sla-record", "name": "Data contract, key, type, nullability, partition, order, compatibility and SLA record", "description": "Immutable or successor-versioned pipeline evidence for data contract, key, type, nullability, partition, order, compatibility and sla.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for data-contract-key-type-nullability-partition-order-compatibility-and-sla; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-003", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "logic-configuration-schedule-and-execution-policy", "name": "Logic, configuration, schedule and execution policy", "description": "Groups governed pipeline context for logic, configuration, schedule and execution policy.", "rationale": "Executable references and policies are governed separately from runtime state.", "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-010", "SRC-012", "SRC-013", "SRC-014", "SRC-015", "SRC-016" ], "layers": [ { "id": "logic-artifacts-parameters-and-runtime-references", "name": "Logic artifacts, parameters and runtime references", "description": "Groups governed pipeline context for logic artifacts, parameters and runtime references.", "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-013", "SRC-014" ], "findings": [ { "id": "code-query-notebook-container-function-artifact-version-and-digest", "name": "Code, query, notebook, container, function, artifact, version and digest", "description": "Records code, query, notebook, container, function, artifact, version and digest as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-013", "SRC-014" ], "questions": [ { "id": "code-query-notebook-container-function-artifact-version-and-digest-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish code, query, notebook, container, function, artifact, version and digest?", "kind": "provenance", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "code-query-notebook-container-function-artifact-version-and-digest-q02", "text": "Who declared, owns, approves, operates, observes or may rely on code, query, notebook, container, function, artifact, version and digest, under which authority and limits?", "kind": "process", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "code-query-notebook-container-function-artifact-version-and-digest-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify code, query, notebook, container, function, artifact, version and digest?", "kind": "decision", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "code-query-notebook-container-function-artifact-version-and-digest-data", "name": "Code, query, notebook, container, function, artifact, version and digest data", "description": "Typed pipeline-definition data for code, query, notebook, container, function, artifact, version and digest, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-013", "SRC-014" ] } ], "artifacts": [ { "id": "code-query-notebook-container-function-artifact-version-and-digest-record", "name": "Code, query, notebook, container, function, artifact, version and digest record", "description": "Immutable or successor-versioned pipeline evidence for code, query, notebook, container, function, artifact, version and digest.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for code-query-notebook-container-function-artifact-version-and-digest; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-013", "SRC-014" ] } ], "inline_only_rationale": null }, { "id": "parameter-configuration-dependency-environment-secret-and-credential-reference", "name": "Parameter, configuration, dependency, environment, secret and credential reference", "description": "Records parameter, configuration, dependency, environment, secret and credential reference as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-013", "SRC-014" ], "questions": [ { "id": "parameter-configuration-dependency-environment-secret-and-credential-reference-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish parameter, configuration, dependency, environment, secret and credential reference?", "kind": "security", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "parameter-configuration-dependency-environment-secret-and-credential-reference-q02", "text": "Who declared, owns, approves, operates, observes or may rely on parameter, configuration, dependency, environment, secret and credential reference, under which authority and limits?", "kind": "validation", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "parameter-configuration-dependency-environment-secret-and-credential-reference-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify parameter, configuration, dependency, environment, secret and credential reference?", "kind": "state", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "parameter-configuration-dependency-environment-secret-and-credential-reference-data", "name": "Parameter, configuration, dependency, environment, secret and credential reference data", "description": "Typed pipeline-definition data for parameter, configuration, dependency, environment, secret and credential reference, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-013", "SRC-014" ] } ], "artifacts": [ { "id": "parameter-configuration-dependency-environment-secret-and-credential-reference-record", "name": "Parameter, configuration, dependency, environment, secret and credential reference record", "description": "Immutable or successor-versioned pipeline evidence for parameter, configuration, dependency, environment, secret and credential reference.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for parameter-configuration-dependency-environment-secret-and-credential-reference; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-013", "SRC-014" ] } ], "inline_only_rationale": null } ] }, { "id": "triggers-schedules-and-execution-semantics", "name": "Triggers, schedules and execution semantics", "description": "Groups governed pipeline context for triggers, schedules and execution semantics.", "source_refs": [ "SRC-002", "SRC-003", "SRC-010", "SRC-012", "SRC-015", "SRC-016" ], "findings": [ { "id": "manual-time-event-dependency-trigger-schedule-timezone-and-data-interval", "name": "Manual, time, event or dependency trigger, schedule, timezone and data interval", "description": "Records manual, time, event or dependency trigger, schedule, timezone and data interval as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-002", "SRC-003", "SRC-010", "SRC-012", "SRC-015", "SRC-016" ], "questions": [ { "id": "manual-time-event-dependency-trigger-schedule-timezone-and-data-interval-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish manual, time, event or dependency trigger, schedule, timezone and data interval?", "kind": "temporal", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "manual-time-event-dependency-trigger-schedule-timezone-and-data-interval-q02", "text": "Who declared, owns, approves, operates, observes or may rely on manual, time, event or dependency trigger, schedule, timezone and data interval, under which authority and limits?", "kind": "privacy", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "manual-time-event-dependency-trigger-schedule-timezone-and-data-interval-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify manual, time, event or dependency trigger, schedule, timezone and data interval?", "kind": "identity", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "manual-time-event-dependency-trigger-schedule-timezone-and-data-interval-data", "name": "Manual, time, event or dependency trigger, schedule, timezone and data interval data", "description": "Typed pipeline-definition data for manual, time, event or dependency trigger, schedule, timezone and data interval, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-002", "SRC-003", "SRC-010", "SRC-012", "SRC-015", "SRC-016" ] } ], "artifacts": [ { "id": "manual-time-event-dependency-trigger-schedule-timezone-and-data-interval-record", "name": "Manual, time, event or dependency trigger, schedule, timezone and data interval record", "description": "Immutable or successor-versioned pipeline evidence for manual, time, event or dependency trigger, schedule, timezone and data interval.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for manual-time-event-dependency-trigger-schedule-timezone-and-data-interval; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-002", "SRC-003", "SRC-010", "SRC-012", "SRC-015", "SRC-016" ] } ], "inline_only_rationale": null }, { "id": "retry-timeout-concurrency-idempotency-checkpoint-backfill-replay-and-catchup", "name": "Retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup", "description": "Records retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-002", "SRC-003", "SRC-010", "SRC-012", "SRC-015", "SRC-016" ], "questions": [ { "id": "retry-timeout-concurrency-idempotency-checkpoint-backfill-replay-and-catchup-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup?", "kind": "process", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "retry-timeout-concurrency-idempotency-checkpoint-backfill-replay-and-catchup-q02", "text": "Who declared, owns, approves, operates, observes or may rely on retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup, under which authority and limits?", "kind": "lifecycle", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "retry-timeout-concurrency-idempotency-checkpoint-backfill-replay-and-catchup-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup?", "kind": "classification", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "retry-timeout-concurrency-idempotency-checkpoint-backfill-replay-and-catchup-data", "name": "Retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup data", "description": "Typed pipeline-definition data for retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-002", "SRC-003", "SRC-010", "SRC-012", "SRC-015", "SRC-016" ] } ], "artifacts": [ { "id": "retry-timeout-concurrency-idempotency-checkpoint-backfill-replay-and-catchup-record", "name": "Retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup record", "description": "Immutable or successor-versioned pipeline evidence for retry, timeout, concurrency, idempotency, checkpoint, backfill, replay and catchup.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for retry-timeout-concurrency-idempotency-checkpoint-backfill-replay-and-catchup; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-002", "SRC-003", "SRC-010", "SRC-012", "SRC-015", "SRC-016" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "lineage-quality-observability-and-data-products", "name": "Lineage, quality, observability and data products", "description": "Groups governed pipeline context for lineage, quality, observability and data products.", "rationale": "Intended lineage and acceptance criteria remain distinct from runtime evidence.", "source_refs": [ "SRC-001", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-011" ], "layers": [ { "id": "intended-lineage-provenance-and-runtime-observation", "name": "Intended lineage, provenance and runtime observation", "description": "Groups governed pipeline context for intended lineage, provenance and runtime observation.", "source_refs": [ "SRC-001", "SRC-005", "SRC-006" ], "findings": [ { "id": "expected-input-output-field-mapping-transform-semantics-and-derivation", "name": "Expected input, output, field mapping, transform semantics and derivation", "description": "Records expected input, output, field mapping, transform semantics and derivation as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-005", "SRC-006" ], "questions": [ { "id": "expected-input-output-field-mapping-transform-semantics-and-derivation-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish expected input, output, field mapping, transform semantics and derivation?", "kind": "provenance", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "expected-input-output-field-mapping-transform-semantics-and-derivation-q02", "text": "Who declared, owns, approves, operates, observes or may rely on expected input, output, field mapping, transform semantics and derivation, under which authority and limits?", "kind": "quality", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "expected-input-output-field-mapping-transform-semantics-and-derivation-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify expected input, output, field mapping, transform semantics and derivation?", "kind": "relationship", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "expected-input-output-field-mapping-transform-semantics-and-derivation-data", "name": "Expected input, output, field mapping, transform semantics and derivation data", "description": "Typed pipeline-definition data for expected input, output, field mapping, transform semantics and derivation, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-005", "SRC-006" ] } ], "artifacts": [ { "id": "expected-input-output-field-mapping-transform-semantics-and-derivation-record", "name": "Expected input, output, field mapping, transform semantics and derivation record", "description": "Immutable or successor-versioned pipeline evidence for expected input, output, field mapping, transform semantics and derivation.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for expected-input-output-field-mapping-transform-semantics-and-derivation; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-005", "SRC-006" ] } ], "inline_only_rationale": null }, { "id": "runtime-lineage-event-source-version-confidence-gap-and-current-view", "name": "Runtime lineage event, source, version, confidence, gap and current view", "description": "Records runtime lineage event, source, version, confidence, gap and current view as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-005", "SRC-006" ], "questions": [ { "id": "runtime-lineage-event-source-version-confidence-gap-and-current-view-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish runtime lineage event, source, version, confidence, gap and current view?", "kind": "evidence", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "runtime-lineage-event-source-version-confidence-gap-and-current-view-q02", "text": "Who declared, owns, approves, operates, observes or may rely on runtime lineage event, source, version, confidence, gap and current view, under which authority and limits?", "kind": "security", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "runtime-lineage-event-source-version-confidence-gap-and-current-view-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify runtime lineage event, source, version, confidence, gap and current view?", "kind": "authority", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "runtime-lineage-event-source-version-confidence-gap-and-current-view-data", "name": "Runtime lineage event, source, version, confidence, gap and current view data", "description": "Typed pipeline-definition data for runtime lineage event, source, version, confidence, gap and current view, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-005", "SRC-006" ] } ], "artifacts": [ { "id": "runtime-lineage-event-source-version-confidence-gap-and-current-view-record", "name": "Runtime lineage event, source, version, confidence, gap and current view record", "description": "Immutable or successor-versioned pipeline evidence for runtime lineage event, source, version, confidence, gap and current view.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for runtime-lineage-event-source-version-confidence-gap-and-current-view; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-005", "SRC-006" ] } ], "inline_only_rationale": null } ] }, { "id": "quality-rules-slos-telemetry-and-cost", "name": "Quality rules, SLOs, telemetry and cost", "description": "Groups governed pipeline context for quality rules, slos, telemetry and cost.", "source_refs": [ "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-011" ], "findings": [ { "id": "validation-rule-dimension-threshold-quarantine-acceptance-and-exception", "name": "Validation rule, dimension, threshold, quarantine, acceptance and exception", "description": "Records validation rule, dimension, threshold, quarantine, acceptance and exception as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-011" ], "questions": [ { "id": "validation-rule-dimension-threshold-quarantine-acceptance-and-exception-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish validation rule, dimension, threshold, quarantine, acceptance and exception?", "kind": "quality", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "validation-rule-dimension-threshold-quarantine-acceptance-and-exception-q02", "text": "Who declared, owns, approves, operates, observes or may rely on validation rule, dimension, threshold, quarantine, acceptance and exception, under which authority and limits?", "kind": "retention", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "validation-rule-dimension-threshold-quarantine-acceptance-and-exception-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify validation rule, dimension, threshold, quarantine, acceptance and exception?", "kind": "requirement", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "validation-rule-dimension-threshold-quarantine-acceptance-and-exception-data", "name": "Validation rule, dimension, threshold, quarantine, acceptance and exception data", "description": "Typed pipeline-definition data for validation rule, dimension, threshold, quarantine, acceptance and exception, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-011" ] } ], "artifacts": [ { "id": "validation-rule-dimension-threshold-quarantine-acceptance-and-exception-record", "name": "Validation rule, dimension, threshold, quarantine, acceptance and exception record", "description": "Immutable or successor-versioned pipeline evidence for validation rule, dimension, threshold, quarantine, acceptance and exception.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for validation-rule-dimension-threshold-quarantine-acceptance-and-exception; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-011" ] } ], "inline_only_rationale": null }, { "id": "log-metric-trace-freshness-latency-volume-error-cost-and-slo", "name": "Log, metric, trace, freshness, latency, volume, error, cost and SLO", "description": "Records log, metric, trace, freshness, latency, volume, error, cost and slo as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-011" ], "questions": [ { "id": "log-metric-trace-freshness-latency-volume-error-cost-and-slo-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish log, metric, trace, freshness, latency, volume, error, cost and slo?", "kind": "measurement", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "log-metric-trace-freshness-latency-volume-error-cost-and-slo-q02", "text": "Who declared, owns, approves, operates, observes or may rely on log, metric, trace, freshness, latency, volume, error, cost and slo, under which authority and limits?", "kind": "interoperability", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "log-metric-trace-freshness-latency-volume-error-cost-and-slo-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify log, metric, trace, freshness, latency, volume, error, cost and slo?", "kind": "constraint", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "log-metric-trace-freshness-latency-volume-error-cost-and-slo-data", "name": "Log, metric, trace, freshness, latency, volume, error, cost and SLO data", "description": "Typed pipeline-definition data for log, metric, trace, freshness, latency, volume, error, cost and slo, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-011" ] } ], "artifacts": [ { "id": "log-metric-trace-freshness-latency-volume-error-cost-and-slo-record", "name": "Log, metric, trace, freshness, latency, volume, error, cost and SLO record", "description": "Immutable or successor-versioned pipeline evidence for log, metric, trace, freshness, latency, volume, error, cost and slo.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for log-metric-trace-freshness-latency-volume-error-cost-and-slo; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-011" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "security-privacy-resilience-and-compliance", "name": "Security, privacy, resilience and compliance", "description": "Groups governed pipeline context for security, privacy, resilience and compliance.", "rationale": "Control intent is explicit while credentials and enforcement remain authoritative elsewhere.", "source_refs": [ "SRC-005", "SRC-006", "SRC-007", "SRC-009", "SRC-011", "SRC-012", "SRC-013", "SRC-014" ], "layers": [ { "id": "identity-access-secrets-zones-and-protection", "name": "Identity, access, secrets, zones and protection", "description": "Groups governed pipeline context for identity, access, secrets, zones and protection.", "source_refs": [ "SRC-009", "SRC-011", "SRC-013", "SRC-014" ], "findings": [ { "id": "service-identity-permission-data-zone-encryption-network-and-least-privilege", "name": "Service identity, permission, data zone, encryption, network and least privilege", "description": "Records service identity, permission, data zone, encryption, network and least privilege as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-009", "SRC-011", "SRC-013", "SRC-014" ], "questions": [ { "id": "service-identity-permission-data-zone-encryption-network-and-least-privilege-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish service identity, permission, data zone, encryption, network and least privilege?", "kind": "security", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "service-identity-permission-data-zone-encryption-network-and-least-privilege-q02", "text": "Who declared, owns, approves, operates, observes or may rely on service identity, permission, data zone, encryption, network and least privilege, under which authority and limits?", "kind": "decision", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "service-identity-permission-data-zone-encryption-network-and-least-privilege-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify service identity, permission, data zone, encryption, network and least privilege?", "kind": "event", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "service-identity-permission-data-zone-encryption-network-and-least-privilege-data", "name": "Service identity, permission, data zone, encryption, network and least privilege data", "description": "Typed pipeline-definition data for service identity, permission, data zone, encryption, network and least privilege, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-009", "SRC-011", "SRC-013", "SRC-014" ] } ], "artifacts": [ { "id": "service-identity-permission-data-zone-encryption-network-and-least-privilege-record", "name": "Service identity, permission, data zone, encryption, network and least privilege record", "description": "Immutable or successor-versioned pipeline evidence for service identity, permission, data zone, encryption, network and least privilege.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for service-identity-permission-data-zone-encryption-network-and-least-privilege; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-009", "SRC-011", "SRC-013", "SRC-014" ] } ], "inline_only_rationale": null }, { "id": "secret-credential-key-reference-rotation-separation-and-exception", "name": "Secret, credential, key reference, rotation, separation and exception", "description": "Records secret, credential, key reference, rotation, separation and exception as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-009", "SRC-011", "SRC-013", "SRC-014" ], "questions": [ { "id": "secret-credential-key-reference-rotation-separation-and-exception-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish secret, credential, key reference, rotation, separation and exception?", "kind": "security", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "secret-credential-key-reference-rotation-separation-and-exception-q02", "text": "Who declared, owns, approves, operates, observes or may rely on secret, credential, key reference, rotation, separation and exception, under which authority and limits?", "kind": "state", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "secret-credential-key-reference-rotation-separation-and-exception-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify secret, credential, key reference, rotation, separation and exception?", "kind": "temporal", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "secret-credential-key-reference-rotation-separation-and-exception-data", "name": "Secret, credential, key reference, rotation, separation and exception data", "description": "Typed pipeline-definition data for secret, credential, key reference, rotation, separation and exception, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-009", "SRC-011", "SRC-013", "SRC-014" ] } ], "artifacts": [ { "id": "secret-credential-key-reference-rotation-separation-and-exception-record", "name": "Secret, credential, key reference, rotation, separation and exception record", "description": "Immutable or successor-versioned pipeline evidence for secret, credential, key reference, rotation, separation and exception.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for secret-credential-key-reference-rotation-separation-and-exception; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-009", "SRC-011", "SRC-013", "SRC-014" ] } ], "inline_only_rationale": null } ] }, { "id": "failure-recovery-rights-retention-and-obligations", "name": "Failure, recovery, rights, retention and obligations", "description": "Groups governed pipeline context for failure, recovery, rights, retention and obligations.", "source_refs": [ "SRC-005", "SRC-006", "SRC-007", "SRC-012", "SRC-014" ], "findings": [ { "id": "failure-mode-detection-rollback-recovery-continuity-disaster-and-residual-risk", "name": "Failure mode, detection, rollback, recovery, continuity, disaster and residual risk", "description": "Records failure mode, detection, rollback, recovery, continuity, disaster and residual risk as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-005", "SRC-006", "SRC-007", "SRC-012", "SRC-014" ], "questions": [ { "id": "failure-mode-detection-rollback-recovery-continuity-disaster-and-residual-risk-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish failure mode, detection, rollback, recovery, continuity, disaster and residual risk?", "kind": "requirement", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "failure-mode-detection-rollback-recovery-continuity-disaster-and-residual-risk-q02", "text": "Who declared, owns, approves, operates, observes or may rely on failure mode, detection, rollback, recovery, continuity, disaster and residual risk, under which authority and limits?", "kind": "identity", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "failure-mode-detection-rollback-recovery-continuity-disaster-and-residual-risk-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify failure mode, detection, rollback, recovery, continuity, disaster and residual risk?", "kind": "composition", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "failure-mode-detection-rollback-recovery-continuity-disaster-and-residual-risk-data", "name": "Failure mode, detection, rollback, recovery, continuity, disaster and residual risk data", "description": "Typed pipeline-definition data for failure mode, detection, rollback, recovery, continuity, disaster and residual risk, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-005", "SRC-006", "SRC-007", "SRC-012", "SRC-014" ] } ], "artifacts": [ { "id": "failure-mode-detection-rollback-recovery-continuity-disaster-and-residual-risk-record", "name": "Failure mode, detection, rollback, recovery, continuity, disaster and residual risk record", "description": "Immutable or successor-versioned pipeline evidence for failure mode, detection, rollback, recovery, continuity, disaster and residual risk.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for failure-mode-detection-rollback-recovery-continuity-disaster-and-residual-risk; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-005", "SRC-006", "SRC-007", "SRC-012", "SRC-014" ] } ], "inline_only_rationale": null }, { "id": "purpose-rights-minimization-residency-retention-deletion-consent-and-legal-hold", "name": "Purpose, rights, minimization, residency, retention, deletion, consent and legal hold", "description": "Records purpose, rights, minimization, residency, retention, deletion, consent and legal hold as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-005", "SRC-006", "SRC-007", "SRC-012", "SRC-014" ], "questions": [ { "id": "purpose-rights-minimization-residency-retention-deletion-consent-and-legal-hold-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish purpose, rights, minimization, residency, retention, deletion, consent and legal hold?", "kind": "privacy", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "purpose-rights-minimization-residency-retention-deletion-consent-and-legal-hold-q02", "text": "Who declared, owns, approves, operates, observes or may rely on purpose, rights, minimization, residency, retention, deletion, consent and legal hold, under which authority and limits?", "kind": "classification", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "purpose-rights-minimization-residency-retention-deletion-consent-and-legal-hold-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify purpose, rights, minimization, residency, retention, deletion, consent and legal hold?", "kind": "evidence", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "purpose-rights-minimization-residency-retention-deletion-consent-and-legal-hold-data", "name": "Purpose, rights, minimization, residency, retention, deletion, consent and legal hold data", "description": "Typed pipeline-definition data for purpose, rights, minimization, residency, retention, deletion, consent and legal hold, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-005", "SRC-006", "SRC-007", "SRC-012", "SRC-014" ] } ], "artifacts": [ { "id": "purpose-rights-minimization-residency-retention-deletion-consent-and-legal-hold-record", "name": "Purpose, rights, minimization, residency, retention, deletion, consent and legal hold record", "description": "Immutable or successor-versioned pipeline evidence for purpose, rights, minimization, residency, retention, deletion, consent and legal hold.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for purpose-rights-minimization-residency-retention-deletion-consent-and-legal-hold; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-005", "SRC-006", "SRC-007", "SRC-012", "SRC-014" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "lifecycle-governance-runs-and-projections", "name": "Lifecycle, governance, runs and projections", "description": "Groups governed pipeline context for lifecycle, governance, runs and projections.", "rationale": "Governed change preserves history and treats runtime and interoperability views as non-owning projections.", "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013", "SRC-014", "SRC-015", "SRC-016" ], "layers": [ { "id": "release-change-deployment-and-run-linkages", "name": "Release, change, deployment and run linkages", "description": "Groups governed pipeline context for release, change, deployment and run linkages.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005", "SRC-012", "SRC-013", "SRC-014" ], "findings": [ { "id": "draft-reviewed-approved-published-deployed-active-paused-deprecated-and-retired", "name": "Draft, reviewed, approved, published, deployed, active, paused, deprecated and retired", "description": "Records draft, reviewed, approved, published, deployed, active, paused, deprecated and retired as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005", "SRC-012", "SRC-013", "SRC-014" ], "questions": [ { "id": "draft-reviewed-approved-published-deployed-active-paused-deprecated-and-retired-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish draft, reviewed, approved, published, deployed, active, paused, deprecated and retired?", "kind": "lifecycle", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "draft-reviewed-approved-published-deployed-active-paused-deprecated-and-retired-q02", "text": "Who declared, owns, approves, operates, observes or may rely on draft, reviewed, approved, published, deployed, active, paused, deprecated and retired, under which authority and limits?", "kind": "relationship", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "draft-reviewed-approved-published-deployed-active-paused-deprecated-and-retired-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify draft, reviewed, approved, published, deployed, active, paused, deprecated and retired?", "kind": "ownership", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "draft-reviewed-approved-published-deployed-active-paused-deprecated-and-retired-data", "name": "Draft, reviewed, approved, published, deployed, active, paused, deprecated and retired data", "description": "Typed pipeline-definition data for draft, reviewed, approved, published, deployed, active, paused, deprecated and retired, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-005", "SRC-012", "SRC-013", "SRC-014" ] } ], "artifacts": [ { "id": "draft-reviewed-approved-published-deployed-active-paused-deprecated-and-retired-record", "name": "Draft, reviewed, approved, published, deployed, active, paused, deprecated and retired record", "description": "Immutable or successor-versioned pipeline evidence for draft, reviewed, approved, published, deployed, active, paused, deprecated and retired.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for draft-reviewed-approved-published-deployed-active-paused-deprecated-and-retired; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005", "SRC-012", "SRC-013", "SRC-014" ] } ], "inline_only_rationale": null }, { "id": "change-impact-compatibility-migration-rollback-deployment-run-incident-and-remediation", "name": "Change impact, compatibility, migration, rollback, deployment, run, incident and remediation", "description": "Records change impact, compatibility, migration, rollback, deployment, run, incident and remediation as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005", "SRC-012", "SRC-013", "SRC-014" ], "questions": [ { "id": "change-impact-compatibility-migration-rollback-deployment-run-incident-and-remediation-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish change impact, compatibility, migration, rollback, deployment, run, incident and remediation?", "kind": "lifecycle", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "change-impact-compatibility-migration-rollback-deployment-run-incident-and-remediation-q02", "text": "Who declared, owns, approves, operates, observes or may rely on change impact, compatibility, migration, rollback, deployment, run, incident and remediation, under which authority and limits?", "kind": "authority", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "change-impact-compatibility-migration-rollback-deployment-run-incident-and-remediation-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify change impact, compatibility, migration, rollback, deployment, run, incident and remediation?", "kind": "measurement", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "change-impact-compatibility-migration-rollback-deployment-run-incident-and-remediation-data", "name": "Change impact, compatibility, migration, rollback, deployment, run, incident and remediation data", "description": "Typed pipeline-definition data for change impact, compatibility, migration, rollback, deployment, run, incident and remediation, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-005", "SRC-012", "SRC-013", "SRC-014" ] } ], "artifacts": [ { "id": "change-impact-compatibility-migration-rollback-deployment-run-incident-and-remediation-record", "name": "Change impact, compatibility, migration, rollback, deployment, run, incident and remediation record", "description": "Immutable or successor-versioned pipeline evidence for change impact, compatibility, migration, rollback, deployment, run, incident and remediation.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for change-impact-compatibility-migration-rollback-deployment-run-incident-and-remediation; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-002", "SRC-005", "SRC-012", "SRC-013", "SRC-014" ] } ], "inline_only_rationale": null } ] }, { "id": "catalog-projection-audit-and-records", "name": "Catalog, projection, audit and records", "description": "Groups governed pipeline context for catalog, projection, audit and records.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013", "SRC-015", "SRC-016" ], "findings": [ { "id": "catalog-dataset-data-product-owner-status-and-discoverability", "name": "Catalog, dataset, data product, owner, status and discoverability", "description": "Records catalog, dataset, data product, owner, status and discoverability as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013", "SRC-015", "SRC-016" ], "questions": [ { "id": "catalog-dataset-data-product-owner-status-and-discoverability-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish catalog, dataset, data product, owner, status and discoverability?", "kind": "classification", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "catalog-dataset-data-product-owner-status-and-discoverability-q02", "text": "Who declared, owns, approves, operates, observes or may rely on catalog, dataset, data product, owner, status and discoverability, under which authority and limits?", "kind": "requirement", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "catalog-dataset-data-product-owner-status-and-discoverability-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify catalog, dataset, data product, owner, status and discoverability?", "kind": "exception", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "catalog-dataset-data-product-owner-status-and-discoverability-data", "name": "Catalog, dataset, data product, owner, status and discoverability data", "description": "Typed pipeline-definition data for catalog, dataset, data product, owner, status and discoverability, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013", "SRC-015", "SRC-016" ] } ], "artifacts": [ { "id": "catalog-dataset-data-product-owner-status-and-discoverability-record", "name": "Catalog, dataset, data product, owner, status and discoverability record", "description": "Immutable or successor-versioned pipeline evidence for catalog, dataset, data product, owner, status and discoverability.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for catalog-dataset-data-product-owner-status-and-discoverability; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013", "SRC-015", "SRC-016" ] } ], "inline_only_rationale": null }, { "id": "bpmn-cwl-airflow-openlineage-prov-dcat-dqv-shacl-cloudevents-and-otel-projection", "name": "BPMN, CWL, Airflow, OpenLineage, PROV, DCAT, DQV, SHACL, CloudEvents and OTel projection", "description": "Records bpmn, cwl, airflow, openlineage, prov, dcat, dqv, shacl, cloudevents and otel projection as governed pipeline-definition context while deployment, scheduler, run, dataset, lineage-event, secret, telemetry and records masters remain external.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013", "SRC-015", "SRC-016" ], "questions": [ { "id": "bpmn-cwl-airflow-openlineage-prov-dcat-dqv-shacl-cloudevents-and-otel-projection-q01", "text": "What stable identity, pipeline version, typed value and explicit unknown establish bpmn, cwl, airflow, openlineage, prov, dcat, dqv, shacl, cloudevents and otel projection?", "kind": "interoperability", "answer_data": [ "pipeline, component and external master identifiers", "typed value, unit, schema or policy profile", "unknown, unavailable, disputed and not-applicable states" ] }, { "id": "bpmn-cwl-airflow-openlineage-prov-dcat-dqv-shacl-cloudevents-and-otel-projection-q02", "text": "Who declared, owns, approves, operates, observes or may rely on bpmn, cwl, airflow, openlineage, prov, dcat, dqv, shacl, cloudevents and otel projection, under which authority and limits?", "kind": "constraint", "answer_data": [ "actor, agent, role, organization and contact", "authority, purpose, policy and access scope", "conflict, exception, escalation and accountability" ] }, { "id": "bpmn-cwl-airflow-openlineage-prov-dcat-dqv-shacl-cloudevents-and-otel-projection-q03", "text": "Which effective, release, deployment, scheduled, event, recorded, ingested and knowledge times, evidence and uncertainty qualify bpmn, cwl, airflow, openlineage, prov, dcat, dqv, shacl, cloudevents and otel projection?", "kind": "provenance", "answer_data": [ "distinct definition, release and runtime times", "source, evidence, method, confidence and uncertainty", "successor, correction, retention and audit" ] } ], "data_elements": [ { "id": "bpmn-cwl-airflow-openlineage-prov-dcat-dqv-shacl-cloudevents-and-otel-projection-data", "name": "BPMN, CWL, Airflow, OpenLineage, PROV, DCAT, DQV, SHACL, CloudEvents and OTel projection data", "description": "Typed pipeline-definition data for bpmn, cwl, airflow, openlineage, prov, dcat, dqv, shacl, cloudevents and otel projection, qualified by version, source, authority, time, evidence and provenance.", "value_kind": "collection", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013", "SRC-015", "SRC-016" ] } ], "artifacts": [ { "id": "bpmn-cwl-airflow-openlineage-prov-dcat-dqv-shacl-cloudevents-and-otel-projection-record", "name": "BPMN, CWL, Airflow, OpenLineage, PROV, DCAT, DQV, SHACL, CloudEvents and OTel projection record", "description": "Immutable or successor-versioned pipeline evidence for bpmn, cwl, airflow, openlineage, prov, dcat, dqv, shacl, cloudevents and otel projection.", "media_or_form": [ "logical pipeline-definition assertion", "topology, contract, policy, release, mapping or audit record" ], "serial": true, "identity_strategy": "Pipeline ID plus independent component, contract, policy, release or artifact ID for bpmn-cwl-airflow-openlineage-prov-dcat-dqv-shacl-cloudevents-and-otel-projection; name, path, date, URL and digest never identify the pipeline alone.", "source_refs": [ "SRC-001", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013", "SRC-015", "SRC-016" ] } ], "inline_only_rationale": null } ] } ] } ] }, "functions": [ { "id": "register-pipeline-definition", "name": "Register a pipeline definition", "description": "Governed operation to register a pipeline definition without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "namespace mandate", "owner", "purpose and initial topology" ], "outputs": [ "stable pipeline head" ], "preconditions": [ "identity, namespace, ownership, purpose and duplicate checks pass" ], "effects": [ "a versioned definition exists without deploying or executing it" ], "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005" ] }, { "id": "compose-or-import-topology", "name": "Compose or import pipeline topology", "description": "Governed operation to compose or import pipeline topology without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "pipeline", "nodes", "edges and mapping profile" ], "outputs": [ "validated topology draft" ], "preconditions": [ "node identity, port, dependency, cycle and mapping checks pass" ], "effects": [ "topology is recorded without invoking runtime components" ], "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004" ] }, { "id": "bind-ports-and-data-contracts", "name": "Bind ports and data contracts", "description": "Governed operation to bind ports and data contracts without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "pipeline topology", "dataset references", "contract profiles" ], "outputs": [ "versioned interface bindings" ], "preconditions": [ "schema, type, key, compatibility, authority and quality checks pass" ], "effects": [ "external datasets and contract masters remain independent" ], "source_refs": [ "SRC-001", "SRC-006", "SRC-007", "SRC-008", "SRC-009" ] }, { "id": "bind-logic-and-runtime-references", "name": "Bind logic, configuration and runtime references", "description": "Governed operation to bind logic, configuration and runtime references without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "pipeline", "logic artifacts", "configuration and secret references" ], "outputs": [ "digest-qualified execution profile" ], "preconditions": [ "artifact provenance, version, digest, dependency, environment and access checks pass" ], "effects": [ "secrets remain unresolved outside an authorized runtime" ], "source_refs": [ "SRC-002", "SRC-003", "SRC-009", "SRC-013", "SRC-014" ] }, { "id": "define-trigger-and-execution-policy", "name": "Define triggers, schedule and execution policy", "description": "Governed operation to define triggers, schedule and execution policy without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "pipeline", "trigger profile", "temporal and retry policy" ], "outputs": [ "versioned execution policy" ], "preconditions": [ "timezone, data interval, retry, concurrency, idempotency and exception checks pass" ], "effects": [ "no schedule is registered and no run is started" ], "source_refs": [ "SRC-002", "SRC-003", "SRC-010", "SRC-012", "SRC-015", "SRC-016" ] }, { "id": "validate-pipeline-before-release", "name": "Validate graph, contracts and controls before release", "description": "Governed operation to validate graph, contracts and controls before release without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "pipeline candidate", "validation profiles", "evidence" ], "outputs": [ "validation report and disposition" ], "preconditions": [ "topology, contract, quality, security, privacy and resilience checks execute under declared profiles" ], "effects": [ "a pass authorizes review but does not prove future runtime outcomes" ], "source_refs": [ "SRC-007", "SRC-008", "SRC-009", "SRC-014" ] }, { "id": "release-deprecate-or-retire", "name": "Release, deprecate or retire a pipeline definition", "description": "Governed operation to release, deprecate or retire a pipeline definition without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "current head", "authorized decision", "change impact and migration plan" ], "outputs": [ "successor lifecycle assertion" ], "preconditions": [ "expected revision, approval, compatibility, rollback and notification checks pass" ], "effects": [ "prior versions remain reconstructable and deployments are not changed automatically" ], "source_refs": [ "SRC-005", "SRC-013", "SRC-014" ] }, { "id": "bind-deployment-and-scheduler", "name": "Bind a deployment and scheduler registration", "description": "Governed operation to bind a deployment and scheduler registration without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "released pipeline", "deployment and scheduler references", "environment profile" ], "outputs": [ "non-owning runtime bindings" ], "preconditions": [ "identity, version, authority, configuration and environment checks pass" ], "effects": [ "runtime systems retain execution and operational authority" ], "source_refs": [ "SRC-001", "SRC-002", "SRC-012", "SRC-013" ] }, { "id": "instantiate-or-reference-run", "name": "Instantiate or reference a pipeline run", "description": "Governed operation to instantiate or reference a pipeline run without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "released pipeline version", "authorized trigger", "input and interval references" ], "outputs": [ "run reference and intended lineage context" ], "preconditions": [ "version, input, trigger, authority, idempotency and time-basis checks pass" ], "effects": [ "the separate run master owns attempts, outputs, status and runtime lineage events" ], "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-005", "SRC-010" ] }, { "id": "query-project-correct-retain-and-audit", "name": "Query, project, correct, retain and audit", "description": "Governed operation to query, project, correct, retain and audit without autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure or records disposition.", "inputs": [ "pipeline", "target view", "access and records policy" ], "outputs": [ "filtered projection, successor correction or audit event" ], "preconditions": [ "purpose, authority, mapping, information loss, hold and retention checks pass" ], "effects": [ "restricted context stays protected and corrections never rewrite released history" ], "source_refs": [ "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-015", "SRC-016" ] } ], "composition": [ { "target": "WM-ACT-003 Process / Workflow", "relation": "REFERENCE", "purpose": "Represent the unfrozen parent boundary without inheritance, mutation, execution or cascade authority.", "required": true, "source_refs": [ "SRC-002", "SRC-003", "SRC-004" ] }, { "target": "WM-ACT-053 Data Processing Job / Pipeline Run", "relation": "REFERENCE", "purpose": "Resolve time-bounded executions while preserving separate definition, deployment, run and task identities.", "required": true, "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-012" ] }, { "target": "WM-DAT-006 Data Lineage", "relation": "REFERENCE", "purpose": "Resolve authoritative runtime provenance without absorbing the lineage master.", "required": true, "source_refs": [ "SRC-001", "SRC-005" ] }, { "target": "Dataset, Data Product, Data Contract, Deployment, Scheduler, Software, Secret, Telemetry, Incident, Evidence, Audit and Records models", "relation": "REFERENCE", "purpose": "Resolve authoritative semantics, state, artifacts, controls and evidence without changing their ownership.", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-005", "SRC-006", "SRC-007", "SRC-009", "SRC-011", "SRC-012", "SRC-013", "SRC-014" ] }, { "target": "BPMN 2.0.2, CWL 1.2.1, Airflow 3.3.0, OpenLineage 1.53.0, PROV-O, DCAT 3, DQV, SHACL, JSON Schema 2020-12, CloudEvents 1.0.2, OpenTelemetry 1.44.0, Kubernetes and OCI 1.1.1", "relation": "ALIGN", "purpose": "Project version-pinned workflow, runtime, lineage, catalog, quality, validation, event, telemetry and packaging views with declared loss.", "required": false, "source_refs": [ "SRC-001", "SRC-002", "SRC-003", "SRC-004", "SRC-005", "SRC-006", "SRC-007", "SRC-008", "SRC-009", "SRC-010", "SRC-011", "SRC-012", "SRC-013" ] } ], "serviceLayers": { "dimension": { "owner_package_requirements": [ "Dimension owner, namespace authority, data governance mandate and accountable data product owner", "Authoritative workflow, pipeline run, dataset, data product, schema, contract, lineage, deployment, scheduler, software, secret, telemetry, incident, remediation, audit and records registries", "Approved topology, schema, compatibility, quality, security, privacy, resilience, change, release, access, retention and interoperability profiles", "Role, delegation, code review, deployment, run, backfill, replay, exception, incident and agent-operation policies" ], "namespace_guidance": "Mint pipeline, version, component, port, edge, contract, policy, release, binding and projection IDs; preserve every external master identifier.", "registry_links": [ "https://ver.cy/models/", "https://ver.cy/model-agent-protocol.md" ] }, "canon_and_patch": { "canonicalization_rules": [ "Canonicalize one reusable pipeline by authoritative pipeline identifier and owner namespace, never by display name, file path, schedule, code repository or deployment alone.", "Keep definition, release, deployment, scheduler registration, run, task attempt, dataset, lineage event and incident independently identifiable." ], "patch_rules": [ "Extensions declare topology, contract, schedule, quality, security, privacy, resilience, deployment and interoperability effects.", "Released versions are immutable; changes create linked successor versions with impact, compatibility, migration and rollback evidence.", "Never silently change source or sink binding, transform logic, schema, key, trigger, schedule, timezone, retry, idempotency, security control or retention policy." ], "compatibility_rules": [ "Ignore additive fields only when pipeline identity, version, topology, contracts, authority, temporal basis, access and provenance survive.", "Every projection pins orchestrator, schema, vocabulary, runtime and mapping versions and declares information loss." ] }, "artifact_rules": { "identity_priority": [ "Authoritative master-system identifier for each pipeline, version, component, contract, policy, release, binding or projection, qualified by issuer, namespace and record kind.", "Governed globally resolvable pipeline IRI.", "Dimension UUID or ULID when neither preceding identifier exists." ], "timestamp_rule": "Use RFC 3339 timestamps with seconds and explicit offset or Z; distinguish effective, released, deployed, scheduled, data-interval, actual-start, actual-end, recorded, ingested and knowledge times whenever they differ.", "serial_naming_rule": "Use {pipeline-id}--{pipeline-version}--{component-contract-policy-release-or-artifact-id}--{artifact-kind}--{revision-id}.", "integrity_rule": "Store digest, media type, record kind, pipeline and component scope, schema and runtime versions, actor, event and knowledge times, confidentiality marking and provenance." }, "policies": [ "The pipeline owns its reusable governed definition and version lineage but not Workflow, Pipeline Run, Dataset, Data Product, Data Lineage, Deployment, Scheduler, Software, Secret, Telemetry, Incident, Evidence, Audit or Records masters.", "Intended topology and static lineage are not runtime proof; run success is not proof of data quality, freshness, publication, downstream consumption or compliance.", "Exactly-once delivery, idempotency, ordering, replay safety and deterministic outputs require explicit scoped guarantees and evidence.", "Agents cannot autonomously deploy, execute, schedule, backfill, replay, rotate credentials, widen access, publish data, delete outputs, close incidents or dispose records without delegated authority." ], "crud": { "read": [ "Resolve current head, topology, ports, contracts, logic references, configuration, trigger policy, quality, lineage expectations, controls, lifecycle, runtime bindings and projection loss under the permitted view." ], "create": [ "Bind stable pipeline identity, owner namespace, purpose, version, topology, ports, data contracts, logic references, policies, source authority and initial state before release." ], "update": [ "Create successor versions for topology, contract, logic, configuration, trigger, quality, security, privacy, resilience, lifecycle and projection changes with reason, authority, expected revision, event time and knowledge time." ], "delete": [ "Apply legal hold, lineage, audit, privacy, security and adopting-Dimension records policy; tombstone only the pipeline view without cascading to datasets, runs, deployments, secrets, evidence or records masters, and let authoritative systems execute physical disposition." ] }, "roles": [ { "name": "Data product owner", "responsibilities": [ "Own purpose, outcome, criticality, consumers, acceptance and accountable use of the pipeline." ] }, { "name": "Data steward", "responsibilities": [ "Own dataset meaning, contracts, quality, lineage, classification, rights and retention profiles." ] }, { "name": "Pipeline author and data engineer", "responsibilities": [ "Maintain topology, transform logic references, parameters, dependencies, tests and versioned change evidence." ] }, { "name": "Platform operator and SRE", "responsibilities": [ "Maintain deployment, scheduler, runtime, observability, capacity, recovery and incident linkages outside the definition." ] }, { "name": "Security and privacy reviewer", "responsibilities": [ "Review identity, secrets, access, zones, encryption, purpose, minimization, residency and deletion controls." ] }, { "name": "Independent reviewer and release authority", "responsibilities": [ "Review topology, contracts, change impact, validation, exceptions, compatibility and release decisions within mandate." ] }, { "name": "Records and interoperability steward", "responsibilities": [ "Own audit, retention, disposition, source pins, mappings and declared projection loss." ] } ], "access": { "default_rule": "Deny credentials, secret values, personal or sensitive data, proprietary logic, security details and restricted runtime evidence unless a purpose-bound policy permits the minimum necessary view.", "scopes": [ "bundle", "layer", "finding", "artifact" ], "exceptions": [ "Declared operations, incident, recovery, security, privacy, legal, audit or data-subject access must cite authority, scope, purpose and time limit and must be logged." ], "audit_requirements": [ "Log actor, agent, role, purpose, pipeline and version, operation, authority, policy, RFC 3339 time, affected nodes or fields, source revision and outcome without duplicating unnecessary restricted data." ] }, "agents_bootstrap": { "filename": "AGENTS.md", "required_fields": [ "Name", "Type", "Specification URL", "Storage type URL", "Interface URL", "Processes URL" ], "read_order": [ "Read Dimension data governance, namespace, workflow, dataset, data product, contract, lineage, security, privacy, access, change, deployment, runtime, incident, retention and agent policies.", "Read this pipeline and linked workflow, run, dataset, contract, lineage, deployment, scheduler, software, secret, telemetry, incident, evidence and records models before mutation." ] } }, "coverage": { "claim": "WM-DAT-005 covers one governed reusable data pipeline definition from identity, topology and data contracts through executable references, trigger policy, intended lineage, quality, observability, security, privacy, resilience, lifecycle, runtime linkages and projections. Engine-specific and jurisdiction-specific profiles plus independent external review remain deferred.", "confidence": "medium", "checklist": [ { "dimension": "identity", "status": "covered", "notes": "Identity is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "classification and direct properties", "status": "covered", "notes": "Classification and direct properties is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "recognition and observation", "status": "covered", "notes": "Recognition and observation is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "capabilities and possible actions", "status": "covered", "notes": "Capabilities and possible actions is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "composition", "status": "covered", "notes": "Composition is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "lifecycle", "status": "covered", "notes": "Lifecycle is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "relationships", "status": "covered", "notes": "Relationships is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "temporal", "status": "covered", "notes": "Temporal is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "spatial", "status": "covered", "notes": "Spatial is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "provenance", "status": "covered", "notes": "Provenance is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "ownership and stewardship", "status": "covered", "notes": "Ownership and stewardship is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "validation and quality", "status": "covered", "notes": "Validation and quality is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "access and privacy", "status": "covered", "notes": "Access and privacy is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "retention and deletion", "status": "covered", "notes": "Retention and deletion is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "interoperability", "status": "covered", "notes": "Interoperability is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." }, { "dimension": "authority and ethics", "status": "covered", "notes": "Authority and ethics is explicit; orchestrator, runtime, data-platform, jurisdiction, contract and independent external review remain held where applicable." } ], "known_omissions": [ "Claude and Grok each timed out on one bounded attempt; no independent external result was admitted.", "The unified parent WM-ACT-003 and candidate relations to WM-ACT-053 and WM-DAT-006 are not approved frozen composition edges.", "Engine-specific task types, expression languages, scheduler semantics, exactly-once guarantees and database transaction behavior require separate profiles.", "Jurisdiction, organization, data sensitivity, residency, privacy, retention and rights requirements require adopting-Dimension profiles." ], "conflicts": [ "Pipeline definition, release, deployment, scheduler registration, pipeline run and task attempt are not interchangeable identities.", "Intended topology and lineage are not proof of runtime events, and run success is not proof of accepted data quality or publication.", "Retry, replay, catchup, idempotency, ordering and exactly-once behavior depend on bounded runtime guarantees and cannot be inferred universally." ], "regional_assumptions": [ "Privacy, residency, retention, deletion, consent, access and evidence duties depend on jurisdiction and organization.", "CWL, BPMN, Airflow, OpenLineage, Kubernetes and OCI are versioned technical profiles and not assumed universal, binding or lossless.", "W3C, JSON Schema, CloudEvents, OpenTelemetry, NIST and RFC sources are versioned profiles whose adoption and stability must be declared." ], "adversarial_checks": [ "Reject a pipeline without stable identity, version, owner, namespace, purpose, topology, ports, contracts, source authority, lifecycle and current head.", "Reject silent overwrites of released topology, transform logic, schema, key, trigger, schedule, timezone, quality rule, security control or retention policy.", "Reject autonomous deployment, execution, scheduling, backfill, replay, credential rotation, access expansion, data publication, output deletion, incident closure or records disposition.", "Reject success, quality, freshness, completeness, exactly-once behavior or downstream acceptance inferred only from a completed run or emitted event.", "Reject a projection that collapses definition, release, deployment, scheduler, run, task, dataset and lineage-event identities or hides information loss." ] }, "researchAdjudication": { "providerMode": "single-provider-waiver", "activeProviders": [ "codex" ], "waivedProviders": [ "claude", "grok" ], "providerPolicy": { "contract_version": "1.0.0", "mode": "single-provider-waiver", "effective_at": "2026-09-06T00:00:00Z", "scope": "Canonical single-stream subject-model research after the six-workstream consolidation", "active_providers": [ "codex" ], "waived_providers": [ { "provider": "claude", "authorized_by": "repository owner", "authorized_at": "2026-09-06T00:00:00Z", "reason": "Claude produced no result on prior 1800-second and 900-second attempts and again timed out on bounded 600-second Sonnet and 300-second Haiku passes. The owner prioritized completion over provider availability." }, { "provider": "grok", "authorized_by": "repository owner", "authorized_at": "2026-09-06T00:00:00Z", "reason": "The repository owner authorized completion without Grok when Grok is unavailable, slow or schema-invalid. Grok may still be attempted as a bounded supplemental reviewer, but its failure never blocks a valid Claude plus no-tools result." } ], "review_rule": "Codex may complete source-grounded fallback research after bounded Claude and Grok attempts fail. It requires a separate no-tools adversarial audit and remains reviewable-draft with a visible absence-of-external-review hold.", "supplemental_provider_attempts": [ { "provider": "claude", "required": false, "maximum_attempts": 1, "failure_policy": "record-and-continue", "admission_rule": "Use only a locally schema-valid result whose sources and boundaries survive adjudication." }, { "provider": "grok", "required": false, "maximum_attempts": 1, "failure_policy": "record-and-continue", "admission_rule": "Use only a locally schema-valid result whose sources and boundaries survive adjudication." } ] }, "boundaryDecision": { "entry_kind": "aggregate", "status": "accepted", "rationale": "The root is a reusable versioned definition aggregate containing components, edges, ports, contracts, policies and release history. It is distinct from each deployment, scheduler registration, run, task attempt, dataset, data product and lineage event." }, "decisions": [ { "concept": "Definition versus runtime", "disposition": "accepted", "rationale": "Pipeline definition, release, deployment, scheduler registration, run and task attempt keep independent identities, owners, states and clocks." }, { "concept": "WM-ACT-003 parent signal", "disposition": "accepted-as-unapproved-reference", "rationale": "The general Process / Workflow parent supplies a boundary only; no approved frozen edge grants inheritance, mutation, execution or cascade authority." }, { "concept": "WM-ACT-053 candidate relation", "disposition": "accepted-as-unapproved-reference", "rationale": "A processing run is external. The candidate CONTAINS relation is not approved and is represented as a required non-owning reference." }, { "concept": "WM-DAT-006 candidate relation", "disposition": "accepted-as-unapproved-reference", "rationale": "Runtime lineage is external. The candidate COMPOSE relation is not approved and is represented as a required non-owning reference." }, { "concept": "Topology and lineage evidence", "disposition": "accepted-with-evidence-boundary", "rationale": "Declared edges and expected derivations are intended state; runtime events and observed provenance require source-qualified evidence." }, { "concept": "Data contracts and quality", "disposition": "accepted-with-profile-hold", "rationale": "Schema, compatibility, keys, quality dimensions, thresholds and acceptance depend on versioned contract and validation profiles." }, { "concept": "Schedule and execution semantics", "disposition": "accepted-with-runtime-hold", "rationale": "Logical date, data interval, actual times, retries, concurrency, idempotency, backfill, replay and catchup must remain explicit and runtime-qualified." }, { "concept": "Exactly-once behavior", "disposition": "rejected-as-universal", "rationale": "Exactly-once, ordering, deterministic output and replay safety cannot be inferred from a pipeline definition or successful run and require scoped runtime guarantees." }, { "concept": "Security and privacy controls", "disposition": "accepted-with-mandatory-hold", "rationale": "Credentials remain external; access, data zones, privacy, residency, minimization, retention and deletion require adopting-Dimension policy." }, { "concept": "Interoperability projections", "disposition": "accepted-with-validation-hold", "rationale": "BPMN, CWL, Airflow, OpenLineage, PROV, DCAT, DQV, SHACL, JSON Schema, CloudEvents, OpenTelemetry, Kubernetes and OCI mappings pin versions and declare information loss." }, { "concept": "Single-provider waiver and no-tools audit", "disposition": "accepted-with-mandatory-hold", "rationale": "One bounded Claude Sonnet and one bounded Grok attempt each timed out after 120 seconds. Codex separately audits the frozen validated result and comparison locally without tools or new research facts; assurance remains reviewable-draft." } ], "publicationHolds": [ "Absence-of-external-review hold: one Claude Sonnet and one Grok attempt for WM-DAT-005 each timed out after 120 seconds; no external research result was admitted.", "Relation hold: the WM-ACT-003 parent signal and candidate relations to WM-ACT-053 and WM-DAT-006 are not approved frozen composition edges and grant no inheritance, ownership, mutation or cascade behavior.", "Runtime-profile hold: engine-specific component types, expression languages, deployment, scheduler, retry, concurrency, checkpoint, backfill, replay and catchup semantics require version-pinned profiles.", "Data-contract hold: schema evolution, compatibility, key, nullability, partition, ordering, quality, freshness and acceptance rules require authoritative versioned contracts.", "Exactly-once hold: exactly-once delivery, deterministic output, ordering, idempotency and replay safety must never be inferred without bounded runtime evidence.", "Security and privacy hold: secrets, credentials, proprietary logic, personal or sensitive data, zones, residency, minimization, consent, retention and deletion require purpose-bound access and adopting-Dimension policy.", "Authority hold: deployment, execution, schedule changes, backfill, replay, credential rotation, access widening, data publication, output deletion, incident closure and records disposition require delegated authority.", "Interoperability hold: BPMN, CWL, Airflow, OpenLineage, PROV, DCAT, DQV, SHACL, JSON Schema, CloudEvents, OpenTelemetry, Kubernetes and OCI projections require release pins, conformance tests and information-loss declarations.", "Source-version hold: platform and semantic-convention sources are versioned snapshots; mixed-stability OpenTelemetry domains and current-at-access documentation require re-verification before canonical promotion.", "Independent external review was explicitly waived by the repository owner; this codex-only result remains a reviewable draft." ], "deferredResearch": [ "Approve or reject the WM-ACT-003, WM-ACT-053 and WM-DAT-006 relationships and register Dataset, Data Product, Data Contract, Deployment, Scheduler, Software, Secret, Telemetry, Incident, Evidence, Audit and Records edges.", "Create orchestrator, runtime, data-platform, database, streaming, transaction, schema-evolution, quality, security, privacy and jurisdiction profiles.", "Test idempotency, checkpoint, retry, replay, backfill, ordering and exactly-once claims against concrete engine and storage combinations.", "Validate organization-specific access, credential, data-zone, residency, minimization, consent, retention, deletion, incident and recovery policies.", "Conformance-test version-pinned BPMN, CWL, Airflow, OpenLineage, PROV, DCAT, DQV, SHACL, JSON Schema, CloudEvents, OpenTelemetry, Kubernetes and OCI mappings and obtain independent external review before canonical promotion." ] }, "statistics": { "sources": 16, "bundles": 6, "layers": 12, "findings": 24, "questions": 72, "artifacts": 24, "functions": 10 } }