# Vercy AI instruction - YAML 1.2 (JSON-compatible) { "vercy": "1.0-draft", "publication": { "status": "published", "adjudicationStatus": "reviewable-draft", "publishableCanonical": false, "generatedAt": "2026-10-21T16:57:48Z", "synthesisSha256": "57a9cfe4cb0a95b942cb6bb3adad6c8c2bad2131408bd4a6124e66cb38db66b1", "providerMode": "single-provider-waiver", "providers": [ "Codex" ], "waivedProviders": [ "Claude", "Grok" ] }, "metaModel": { "id": "WM-AI-012", "registryId": "vr.wm-ai-012", "name": "Benchmark Result", "version": "0.1.0", "previousVersions": [], "entryKind": "entity", "family": "World Models", "category": "Information and virtual systems", "industry": [ "Cross-industry" ], "domain": [ "INF.AI.RES" ], "tags": [ "benchmark", "result", "inf.ai.res" ], "status": "published" }, "canonicalUrl": "https://ver.cy/models/wm-ai-012-benchmark-result/", "sourceUrl": "https://github.com/ver-cy/world-models/tree/feat/mega-model-registry/research/runs/wm-ai-012", "model": { "registry_id": "vr.wm-ai-012", "model_id": "WM-AI-012", "name": "Benchmark Result", "entry_kind": "entity", "purpose": "Represent a reported benchmark outcome with the context, reproduction evidence and qualified verification needed to interpret it.", "scope_statement": "One persistent benchmark result record for a specified tested subject, benchmark revision, evaluation scope and reported outcome. A record may carry a metric vector or an explicitly defined aggregate. Distinct runs and replications remain separately identified; corrections create revisions. Missing or failed outcomes are retained without inventing numeric scores.", "in_scope": [ "Result identity, issuer, revision and reporting status", "Pinned subject, benchmark, dataset split and execution references", "Metric semantics, values, aggregation, missingness and uncertainty", "Reproduction manifest and evidence availability", "Review assertions, independence disclosures, objections, comparison conditions and release restrictions" ], "out_of_scope": [ "Executing or scheduling evaluations, training or tuning tested systems", "Mastering benchmark datasets, protocols, model artifacts or hardware assets", "Operating a leaderboard, certifying safety or authorizing deployment", "Executing third-party reproduction instructions or operational attacks" ], "boundary_notes": [ { "neighbor": "WM-AI-003", "distinction": "Registry parent is a navigation/context candidate for evaluation. A result is an output entity, not the evaluation activity or its process engine; link an evaluation reference without inheriting its execution lifecycle.", "source_refs": [ "SRC-001", "SRC-004" ] }, { "neighbor": "WM-AI-009", "distinction": "Dataset and benchmark definition revisions are external masters. Store the used split, digest and protocol binding; do not manage dataset curation or benchmark releases here.", "source_refs": [ "SRC-003", "SRC-006" ] }, { "neighbor": "WM-SFT-004", "distinction": "Reference the tested model artifact and configuration as known. The result neither edits weights nor owns the tested system lifecycle. These neighbor bindings are proposed and unpinned.", "source_refs": [ "SRC-001", "SRC-007" ] }, { "neighbor": "Leaderboard and assurance decision", "distinction": "Ranking and deployment decisions consume results under their own rules. A high score, digest, submission acceptance or reviewer statement alone does not prove general capability, safety or independent reproduction.", "source_refs": [ "SRC-004", "SRC-007" ] } ] }, "sources": [ { "id": "SRC-001", "title": "PROV-O: The PROV Ontology", "organization": "W3C", "url": "https://www.w3.org/TR/prov-o/", "version_or_date": "Recommendation 2013-04-30", "source_type": "ontology", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T16:56:36Z", "relevance": "Entity, activity, attribution, derivation and revision distinctions; conceptual alignment only." }, { "id": "SRC-002", "title": "Data on the Web Best Practices: Data Quality Vocabulary", "organization": "W3C", "url": "https://www.w3.org/TR/vocab-dqv/", "version_or_date": "Working Group Note; edition not independently pinned; accessed 2026-10-21", "source_type": "ontology", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T16:56:36Z", "relevance": "Metric, measurement value, unit and measured resource distinctions. Dataset-quality vocabulary is an analogy, not a universal AI benchmark schema." }, { "id": "SRC-003", "title": "Data Catalog Vocabulary (DCAT) - Version 3", "organization": "W3C", "url": "https://www.w3.org/TR/vocab-dcat-3/", "version_or_date": "Recommendation 2024-08-22", "source_type": "standard", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T16:56:36Z", "relevance": "Dataset/distribution references, versions and checksums for result evidence. Does not establish benchmark validity." }, { "id": "SRC-004", "title": "AI Risk Management Framework 1.0: Core", "organization": "NIST", "url": "https://airc.nist.gov/airmf-resources/airmf/5-sec-core/", "version_or_date": "AI RMF 1.0 (2023), web excerpt; revision in progress noted by source", "source_type": "public-authority", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T16:56:36Z", "relevance": "MAP 2.3 and MEASURE 1-4 support contextual evaluation, documented methods, limitations and independent assessment. Voluntary guidance, not certification." }, { "id": "SRC-005", "title": "Essentials of expressing measurement uncertainty: Basic definitions", "organization": "NIST", "url": "https://physics.nist.gov/cuu/Uncertainty/basic.html", "version_or_date": "Web summary adapted from Technical Note 1297; retrieved 2026-10-21", "source_type": "public-authority", "primary_source": true, "authority_tier": 1, "accessed_at": "2026-10-21T16:56:36Z", "relevance": "Measurement model and uncertainty components. Statistical treatment of benchmark samples still needs a task-specific method." }, { "id": "SRC-006", "title": "MLPerf Inference Submission Guide", "organization": "MLCommons", "url": "https://docs.mlcommons.org/inference/submission/", "version_or_date": "Living documentation retrieved 2026-10-21; release pin unresolved", "source_type": "first-party-doc", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T16:56:36Z", "relevance": "Scenario, division, configuration, accuracy checking and submission package distinctions; one open benchmark profile, not a universal protocol." }, { "id": "SRC-007", "title": "General MLPerf Submission Rules", "organization": "MLCommons", "url": "https://github.com/mlcommons/policies/blob/master/submission_rules.adoc", "version_or_date": "Mutable master branch retrieved 2026-10-21; commit pin unresolved", "source_type": "first-party-doc", "primary_source": true, "authority_tier": 2, "accessed_at": "2026-10-21T16:56:36Z", "relevance": "Sections 5.5-5.11 and 6-8: result logs, configuration identification, replication material, review, objections and withdrawal. Selected sections only; profile rules remain version-dependent." } ], "structure": { "bundles": [ { "id": "bundle-identity", "name": "Identity and claim", "description": "Result-specific context for identity and claim.", "rationale": "A separate concern needed to interpret and govern a reported result without absorbing external evaluation or dataset masters.", "source_refs": [ "SRC-001", "SRC-007" ], "layers": [ { "id": "layer-result-identity", "name": "Result identity", "description": "Assign the result a stable identity independent of display score and file name; record issuer and scope before admitting a claim.", "source_refs": [ "SRC-001", "SRC-007" ], "findings": [ { "id": "finding-result-identity", "name": "Reported assertion identity", "description": "Assign the result a stable identity independent of display score and file name; record issuer and scope before admitting a claim.", "source_refs": [ "SRC-001", "SRC-007" ], "questions": [ { "id": "finding-result-identity-q1", "text": "Which master identifier and revision identify this reported result?", "kind": "identity", "answer_data": [ "master_id", "revision_id", "aliases" ] }, { "id": "finding-result-identity-q2", "text": "Who issued this result and under which reporting authority?", "kind": "authority", "answer_data": [ "issuer_ref", "authority_basis", "asserted_at" ] }, { "id": "finding-result-identity-q3", "text": "Is the outcome an individual run, a metric vector or a declared aggregate?", "kind": "classification", "answer_data": [ "result_kind", "member_result_refs", "aggregation_ref" ] }, { "id": "finding-result-identity-q4", "text": "Which evaluation record produced the result and how is that relationship evidenced?", "kind": "relationship", "answer_data": [ "evaluation_ref", "generation_evidence", "unresolved_reason" ] } ], "data_elements": [ { "id": "finding-result-identity-data-identity", "name": "Result identity", "description": "Master result identifier; revisions and aliases are separately recorded.", "value_kind": "identifier", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-007" ] }, { "id": "finding-result-identity-data-claim-context", "name": "Assertion context", "description": "Issuer, authority basis, result kind, revision, evaluation reference or explicit unresolved status.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-007" ] } ], "artifacts": [ { "id": "finding-result-identity-artifact", "name": "Result assertion record", "description": "Versioned evidence projection for reported assertion identity. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-001", "SRC-007" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-subject-binding", "name": "Subject binding", "description": "Bind reported measurements to the actual tested revision and effective configuration, including transformations; a display label is insufficient.", "source_refs": [ "SRC-001", "SRC-007" ], "findings": [ { "id": "finding-subject-binding", "name": "Tested subject snapshot", "description": "Bind reported measurements to the actual tested revision and effective configuration, including transformations; a display label is insufficient.", "source_refs": [ "SRC-001", "SRC-007" ], "questions": [ { "id": "finding-subject-binding-q1", "text": "Which tested artifact revision or endpoint snapshot does the result describe?", "kind": "identity", "answer_data": [ "subject_ref", "revision_or_digest", "endpoint_observed_at" ] }, { "id": "finding-subject-binding-q2", "text": "Which preprocessing, prompts, tools or runtime configuration were included in the tested boundary?", "kind": "composition", "answer_data": [ "configuration_manifest_ref", "included_components", "excluded_components" ] }, { "id": "finding-subject-binding-q3", "text": "What transformations or tuning separate the tested subject from its declared base?", "kind": "provenance", "answer_data": [ "base_ref", "transformation_refs", "calibration_or_tuning_declaration" ] }, { "id": "finding-subject-binding-q4", "text": "What subject identity information is unavailable or cannot be pinned?", "kind": "exception", "answer_data": [ "unknown_fields", "access_constraints", "comparability_limit" ] } ], "data_elements": [ { "id": "finding-subject-binding-data-subject", "name": "Tested subject binding", "description": "Authoritative tested subject reference or explicitly unresolved supplied identifier.", "value_kind": "reference", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-007" ] }, { "id": "finding-subject-binding-data-configuration", "name": "Tested configuration", "description": "Effective revision, transformations, test boundary and identity gaps.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-007" ] } ], "artifacts": [ { "id": "finding-subject-binding-artifact", "name": "Subject configuration manifest", "description": "Versioned evidence projection for tested subject snapshot. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-001", "SRC-007" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-protocol", "name": "Protocol and sample", "description": "Result-specific context for protocol and sample.", "rationale": "A separate concern needed to interpret and govern a reported result without absorbing external evaluation or dataset masters.", "source_refs": [ "SRC-004", "SRC-006", "SRC-003" ], "layers": [ { "id": "layer-benchmark-binding", "name": "Benchmark binding", "description": "Keep task, metric rules, scenario and benchmark revision together. Profile-specific acceptance rules must be supplied rather than assumed from a benchmark name.", "source_refs": [ "SRC-004", "SRC-006" ], "findings": [ { "id": "finding-benchmark-binding", "name": "Versioned evaluation contract", "description": "Keep task, metric rules, scenario and benchmark revision together. Profile-specific acceptance rules must be supplied rather than assumed from a benchmark name.", "source_refs": [ "SRC-004", "SRC-006" ], "questions": [ { "id": "finding-benchmark-binding-q1", "text": "Which benchmark release and task define the meaning of this outcome?", "kind": "definition", "answer_data": [ "benchmark_ref", "release", "task_id", "protocol_digest" ] }, { "id": "finding-benchmark-binding-q2", "text": "Which scenario, track and permitted modification class apply?", "kind": "classification", "answer_data": [ "scenario", "track", "division", "profile_version" ] }, { "id": "finding-benchmark-binding-q3", "text": "Which accuracy or quality constraints condition reporting of the primary score?", "kind": "requirement", "answer_data": [ "thresholds", "comparator_direction", "constraint_verdict_refs" ] }, { "id": "finding-benchmark-binding-q4", "text": "Which deviations from the declared protocol were used or remain unresolved?", "kind": "exception", "answer_data": [ "deviations", "authorization_refs", "unresolved_requirements" ] } ], "data_elements": [ { "id": "finding-benchmark-binding-data-protocol", "name": "Benchmark contract binding", "description": "Versioned external protocol, scenario and conditional acceptance criteria; no protocol authoring here.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-004", "SRC-006" ] } ], "artifacts": [ { "id": "finding-benchmark-binding-artifact", "name": "Protocol binding record", "description": "Versioned evidence projection for versioned evaluation contract. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-004", "SRC-006" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-sample-binding", "name": "Sample binding", "description": "Describe the actual evaluated subset and exclusions without taking ownership of the dataset. Contamination claims require evidence and can remain unknown.", "source_refs": [ "SRC-003", "SRC-004", "SRC-006" ], "findings": [ { "id": "finding-sample-binding", "name": "Evaluated population and split", "description": "Describe the actual evaluated subset and exclusions without taking ownership of the dataset. Contamination claims require evidence and can remain unknown.", "source_refs": [ "SRC-003", "SRC-004", "SRC-006" ], "questions": [ { "id": "finding-sample-binding-q1", "text": "Which dataset version, split and subset were actually evaluated?", "kind": "relationship", "answer_data": [ "dataset_ref", "distribution_digest", "split", "subset_manifest" ] }, { "id": "finding-sample-binding-q2", "text": "How many eligible, attempted, completed and excluded cases contributed to the result?", "kind": "measurement", "answer_data": [ "eligible_count", "attempted_count", "completed_count", "excluded_count", "denominator_rule" ] }, { "id": "finding-sample-binding-q3", "text": "What evidence addresses overlap with training, tuning or calibration material?", "kind": "quality", "answer_data": [ "overlap_assessment_ref", "assessment_scope", "status", "unresolved_risk" ] }, { "id": "finding-sample-binding-q4", "text": "Which population, language or domain limits restrict generalization from this sample?", "kind": "constraint", "answer_data": [ "population_scope", "language", "domain", "sampling_limits" ] } ], "data_elements": [ { "id": "finding-sample-binding-data-sample", "name": "Evaluated sample binding", "description": "Dataset revision, split, sample manifest, counts and declared population boundaries.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-003", "SRC-004", "SRC-006" ] }, { "id": "finding-sample-binding-data-overlap", "name": "Contamination assessment reference", "description": "Evidence-based overlap claims with assessor, scope and unknown status; absence of evidence is not cleanliness.", "value_kind": "object", "cardinality": "0..n", "required": false, "source_refs": [ "SRC-003", "SRC-004", "SRC-006" ] } ], "artifacts": [ { "id": "finding-sample-binding-artifact", "name": "Sample accounting manifest", "description": "Versioned evidence projection for evaluated population and split. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-003", "SRC-004", "SRC-006" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-execution", "name": "Execution evidence", "description": "Result-specific context for execution evidence.", "rationale": "A separate concern needed to interpret and govern a reported result without absorbing external evaluation or dataset masters.", "source_refs": [ "SRC-001", "SRC-006", "SRC-007", "SRC-003" ], "layers": [ { "id": "layer-run-context", "name": "Run context", "description": "Reference execution events and capture the conditions needed to interpret measurements. This record does not start, stop or change a run.", "source_refs": [ "SRC-001", "SRC-006", "SRC-007" ], "findings": [ { "id": "finding-run-context", "name": "Observed execution conditions", "description": "Reference execution events and capture the conditions needed to interpret measurements. This record does not start, stop or change a run.", "source_refs": [ "SRC-001", "SRC-006", "SRC-007" ], "questions": [ { "id": "finding-run-context-q1", "text": "When did execution, result issuance and local ingestion occur?", "kind": "temporal", "answer_data": [ "run_start", "run_end", "issued_at", "ingested_at", "clock_basis" ] }, { "id": "finding-run-context-q2", "text": "Which hardware and software configuration governed the measured execution?", "kind": "composition", "answer_data": [ "system_manifest_ref", "dependency_versions", "topology", "precision_mode" ] }, { "id": "finding-run-context-q3", "text": "Which load, warm-up, randomization and repetition settings were applied?", "kind": "process", "answer_data": [ "load_profile", "warmup_policy", "seeds", "repetitions", "scheduling_notes" ] }, { "id": "finding-run-context-q4", "text": "Where was the timing or resource measurement boundary drawn?", "kind": "measurement", "answer_data": [ "measurement_boundary", "instrument_ref", "units", "network_inclusion", "calibration_status" ] } ], "data_elements": [ { "id": "finding-run-context-data-execution", "name": "Execution context", "description": "External run references and observed time, configuration, load and measurement boundary; unavailable facts explicit.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-006", "SRC-007" ] } ], "artifacts": [ { "id": "finding-run-context-artifact", "name": "Execution evidence index", "description": "Versioned evidence projection for observed execution conditions. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-001", "SRC-006", "SRC-007" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-reproduction", "name": "Reproduction", "description": "Index the evidence needed for a permitted rerun while preserving access and integrity limitations. An accessible recipe is not proof of successful reproduction.", "source_refs": [ "SRC-003", "SRC-007" ], "findings": [ { "id": "finding-reproduction", "name": "Reproduction material and accessibility", "description": "Index the evidence needed for a permitted rerun while preserving access and integrity limitations. An accessible recipe is not proof of successful reproduction.", "source_refs": [ "SRC-003", "SRC-007" ], "questions": [ { "id": "finding-reproduction-q1", "text": "Which versioned inputs, configuration and logs support a reproduction attempt?", "kind": "evidence", "answer_data": [ "artifact_refs", "versions", "digests", "log_refs" ] }, { "id": "finding-reproduction-q2", "text": "Which required artifacts are available, restricted, expired or missing?", "kind": "access", "answer_data": [ "availability_by_artifact", "access_procedure", "missing_items", "licenses" ] }, { "id": "finding-reproduction-q3", "text": "Which post-processing steps connect raw outputs to the reported score?", "kind": "provenance", "answer_data": [ "raw_output_ref", "scorer_revision", "transformation_graph", "derived_output_ref" ] }, { "id": "finding-reproduction-q4", "text": "What approval and isolation policy governs any future use of supplied reproduction instructions?", "kind": "security", "answer_data": [ "execution_policy_ref", "trust_status", "approval_role", "resource_limits" ] } ], "data_elements": [ { "id": "finding-reproduction-data-reproduction", "name": "Reproduction manifest", "description": "Artifact references, versions, digests, availability, licenses and derivation steps; referenced instructions remain untrusted data.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-003", "SRC-007" ] } ], "artifacts": [ { "id": "finding-reproduction-artifact", "name": "Reproduction evidence manifest", "description": "Versioned evidence projection for reproduction material and accessibility. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-003", "SRC-007" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-measurement", "name": "Measurements and reliability", "description": "Result-specific context for measurements and reliability.", "rationale": "A separate concern needed to interpret and govern a reported result without absorbing external evaluation or dataset masters.", "source_refs": [ "SRC-002", "SRC-004", "SRC-006", "SRC-005" ], "layers": [ { "id": "layer-metric-values", "name": "Metric values", "description": "Represent each metric with its definition, unit, direction and validity state. A failed, missing or nonapplicable value must not become numeric zero.", "source_refs": [ "SRC-002", "SRC-004", "SRC-006" ], "findings": [ { "id": "finding-metric-values", "name": "Typed result values", "description": "Represent each metric with its definition, unit, direction and validity state. A failed, missing or nonapplicable value must not become numeric zero.", "source_refs": [ "SRC-002", "SRC-004", "SRC-006" ], "questions": [ { "id": "finding-metric-values-q1", "text": "Which metric definition, scale, unit and preferred direction apply to each value?", "kind": "definition", "answer_data": [ "metric_ref", "definition_revision", "value_type", "unit", "direction" ] }, { "id": "finding-metric-values-q2", "text": "What value and precision were reported for each metric and sample slice?", "kind": "measurement", "answer_data": [ "values", "slice_refs", "significant_digits", "rounding_rule" ] }, { "id": "finding-metric-values-q3", "text": "Which metrics are measured, missing, failed, censored or not applicable?", "kind": "state", "answer_data": [ "value_status", "reason", "bound_type", "bounds" ] }, { "id": "finding-metric-values-q4", "text": "Which scorer and rule evidence supports the value or its rejection?", "kind": "validation", "answer_data": [ "scorer_ref", "validation_ref", "rule_revision", "verdict", "evidence_ref" ] } ], "data_elements": [ { "id": "finding-metric-values-data-metrics", "name": "Metric outcomes", "description": "Metric rows with typed value or explicit absence, unit, scale, slice, precision, status and evidence.", "value_kind": "collection", "cardinality": "1..n", "required": true, "source_refs": [ "SRC-002", "SRC-004", "SRC-006" ] } ], "artifacts": [ { "id": "finding-metric-values-artifact", "name": "Metric result table", "description": "Versioned evidence projection for typed result values. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-002", "SRC-004", "SRC-006" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-aggregation", "name": "Aggregation and uncertainty", "description": "Preserve the reduction rule and included attempts. Distinguish variation across runs from sampling uncertainty and systematic limitations; no universal interval method is prescribed.", "source_refs": [ "SRC-002", "SRC-004", "SRC-005" ], "findings": [ { "id": "finding-aggregation", "name": "Aggregation and uncertainty account", "description": "Preserve the reduction rule and included attempts. Distinguish variation across runs from sampling uncertainty and systematic limitations; no universal interval method is prescribed.", "source_refs": [ "SRC-002", "SRC-004", "SRC-005" ], "questions": [ { "id": "finding-aggregation-q1", "text": "Which runs or component scores were included and excluded by the aggregation rule?", "kind": "composition", "answer_data": [ "member_refs", "inclusion_rule", "exclusion_reasons", "failed_attempt_refs" ] }, { "id": "finding-aggregation-q2", "text": "How were weighting, normalization and denominators used to derive the aggregate?", "kind": "measurement", "answer_data": [ "aggregation_formula_ref", "weights", "normalization_baseline", "denominators" ] }, { "id": "finding-aggregation-q3", "text": "Which uncertainty estimates accompany the score and what do they mean?", "kind": "measurement", "answer_data": [ "interval", "level", "uncertainty_type", "method", "assumptions", "sample_size" ] }, { "id": "finding-aggregation-q4", "text": "What evidence addresses dependence, selection bias and unresolved uncertainty?", "kind": "quality", "answer_data": [ "dependence_notes", "selection_policy", "sensitivity_ref", "uncertainty_gaps" ] } ], "data_elements": [ { "id": "finding-aggregation-data-aggregation", "name": "Aggregation account", "description": "Required when an aggregate is reported: member results, reduction rule, weights, normalization and exclusions.", "value_kind": "object", "cardinality": "0..1", "required": false, "source_refs": [ "SRC-002", "SRC-004", "SRC-005" ] }, { "id": "finding-aggregation-data-uncertainty", "name": "Uncertainty account", "description": "Estimate and method or explicit not-estimated status; sampling, run variability and systematic limitations distinguished.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-002", "SRC-004", "SRC-005" ] } ], "artifacts": [ { "id": "finding-aggregation-artifact", "name": "Aggregation and uncertainty statement", "description": "Versioned evidence projection for aggregation and uncertainty account. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-002", "SRC-004", "SRC-005" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-assurance", "name": "Verification and challenge", "description": "Result-specific context for verification and challenge.", "rationale": "A separate concern needed to interpret and govern a reported result without absorbing external evaluation or dataset masters.", "source_refs": [ "SRC-001", "SRC-004", "SRC-007" ], "layers": [ { "id": "layer-verification", "name": "Verification", "description": "Store separate assertions for local validation, external review and reproduced outcomes. Independence is a disclosed relationship with evidence, not a boolean inferred from a badge.", "source_refs": [ "SRC-001", "SRC-004", "SRC-007" ], "findings": [ { "id": "finding-verification", "name": "Scoped verification assertions", "description": "Store separate assertions for local validation, external review and reproduced outcomes. Independence is a disclosed relationship with evidence, not a boolean inferred from a badge.", "source_refs": [ "SRC-001", "SRC-004", "SRC-007" ], "questions": [ { "id": "finding-verification-q1", "text": "What verification status applies to each result revision and metric scope?", "kind": "state", "answer_data": [ "verification_status", "result_revision", "metric_scope", "assertion_ref" ] }, { "id": "finding-verification-q2", "text": "Who made the verification assertion and what relationship do they have to the issuer?", "kind": "authority", "answer_data": [ "reviewer_ref", "affiliation_relationship", "conflict_declaration", "independence_basis" ] }, { "id": "finding-verification-q3", "text": "Was verification a metadata check, code inspection or an actual reproduction?", "kind": "evidence", "answer_data": [ "method", "inspected_artifacts", "reproduction_result_ref", "evidence", "performed_at" ] }, { "id": "finding-verification-q4", "text": "Which tolerances and acceptance criteria support the verification conclusion?", "kind": "validation", "answer_data": [ "criteria_ref", "tolerance", "observed_difference", "scope_exclusions", "conclusion" ] } ], "data_elements": [ { "id": "finding-verification-data-verification", "name": "Verification assertions", "description": "Scoped assertions including unverified status; separate checking methods, reviewer relationship and evidence. No verifier is fabricated.", "value_kind": "collection", "cardinality": "1..n", "required": true, "source_refs": [ "SRC-001", "SRC-004", "SRC-007" ] } ], "artifacts": [ { "id": "finding-verification-artifact", "name": "Verification assertion register", "description": "Versioned evidence projection for scoped verification assertions. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-001", "SRC-004", "SRC-007" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-challenge", "name": "Challenges", "description": "Keep objections and conflicting observations linked to the challenged revision. Correction, withdrawal and later reproduction have distinct meanings.", "source_refs": [ "SRC-001", "SRC-007" ], "findings": [ { "id": "finding-challenge", "name": "Contrary evidence and correction", "description": "Keep objections and conflicting observations linked to the challenged revision. Correction, withdrawal and later reproduction have distinct meanings.", "source_refs": [ "SRC-001", "SRC-007" ], "questions": [ { "id": "finding-challenge-q1", "text": "What discrepancy or objection challenges this result and which evidence supports it?", "kind": "evidence", "answer_data": [ "challenge_ref", "affected_revision", "claim", "evidence_refs" ] }, { "id": "finding-challenge-q2", "text": "Which accountable role resolved the challenge and on what recorded basis?", "kind": "decision", "answer_data": [ "decision_ref", "reviewer_role", "disposition", "reasons", "unresolved_items" ] }, { "id": "finding-challenge-q3", "text": "Does the resolution require correction, supersession, withdrawal or no change?", "kind": "lifecycle", "answer_data": [ "transition", "replacement_ref", "effective_at", "reason" ] }, { "id": "finding-challenge-q4", "text": "How can a reader trace both the original assertion and the corrected result?", "kind": "provenance", "answer_data": [ "prior_revision_ref", "revision_chain", "changed_fields", "retained_evidence" ] } ], "data_elements": [ { "id": "finding-challenge-data-challenges", "name": "Challenge dispositions", "description": "Contrary evidence and externally authorized resolution references; unresolved challenges remain visible.", "value_kind": "collection", "cardinality": "0..n", "required": false, "source_refs": [ "SRC-001", "SRC-007" ] }, { "id": "finding-challenge-data-continuity", "name": "Result continuity", "description": "Current revision, prior revision links and withdrawal or correction state.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-001", "SRC-007" ] } ], "artifacts": [ { "id": "finding-challenge-artifact", "name": "Challenge and correction record", "description": "Versioned evidence projection for contrary evidence and correction. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-001", "SRC-007" ] } ], "inline_only_rationale": null } ] } ] }, { "id": "bundle-use", "name": "Interpretation and release", "description": "Result-specific context for interpretation and release.", "rationale": "A separate concern needed to interpret and govern a reported result without absorbing external evaluation or dataset masters.", "source_refs": [ "SRC-004", "SRC-005", "SRC-006", "SRC-003", "SRC-007" ], "layers": [ { "id": "layer-comparability", "name": "Comparability", "description": "Comparison needs compatible task, metric, data and execution context. The result can carry an assessment reference; it does not run a leaderboard or establish deployment fitness.", "source_refs": [ "SRC-004", "SRC-005", "SRC-006" ], "findings": [ { "id": "finding-comparability", "name": "Qualified comparison", "description": "Comparison needs compatible task, metric, data and execution context. The result can carry an assessment reference; it does not run a leaderboard or establish deployment fitness.", "source_refs": [ "SRC-004", "SRC-005", "SRC-006" ], "questions": [ { "id": "finding-comparability-q1", "text": "Which baseline result revision is proposed for comparison?", "kind": "relationship", "answer_data": [ "baseline_result_ref", "baseline_revision", "comparison_purpose" ] }, { "id": "finding-comparability-q2", "text": "Which differences in protocol, data, configuration or measurement boundary limit comparison?", "kind": "constraint", "answer_data": [ "compatibility_checks", "material_differences", "comparison_scope" ] }, { "id": "finding-comparability-q3", "text": "How was any reported improvement computed and with what uncertainty?", "kind": "measurement", "answer_data": [ "delta_formula", "absolute_or_relative", "zero_baseline_policy", "uncertainty_ref" ] }, { "id": "finding-comparability-q4", "text": "What claim is supported by this comparison and which uses remain unjustified?", "kind": "decision", "answer_data": [ "supported_claim", "applicability", "unsupported_uses", "assessment_ref" ] } ], "data_elements": [ { "id": "finding-comparability-data-comparison", "name": "Comparison assertions", "description": "Baseline-bound comparisons with compatibility assessment, uncertainty and explicit unresolved or incomparable status.", "value_kind": "collection", "cardinality": "0..n", "required": false, "source_refs": [ "SRC-004", "SRC-005", "SRC-006" ] } ], "artifacts": [ { "id": "finding-comparability-artifact", "name": "Comparison context statement", "description": "Versioned evidence projection for qualified comparison. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-004", "SRC-005", "SRC-006" ] } ], "inline_only_rationale": null } ] }, { "id": "layer-release", "name": "Release and retention", "description": "Release permissions apply separately to summary, logs, inputs and protected slices. Retention and correction notices must preserve useful provenance without requiring indefinite payload storage.", "source_refs": [ "SRC-003", "SRC-004", "SRC-007" ], "findings": [ { "id": "finding-release", "name": "Disclosure and continued availability", "description": "Release permissions apply separately to summary, logs, inputs and protected slices. Retention and correction notices must preserve useful provenance without requiring indefinite payload storage.", "source_refs": [ "SRC-003", "SRC-004", "SRC-007" ], "questions": [ { "id": "finding-release-q1", "text": "Which accountable custodian governs this result and its evidence copies?", "kind": "ownership", "answer_data": [ "custodian_role", "master_system", "evidence_controllers" ] }, { "id": "finding-release-q2", "text": "Which result fields or artifacts need restricted disclosure or redaction?", "kind": "privacy", "answer_data": [ "classification", "field_scope", "privacy_basis", "disclosure_view" ] }, { "id": "finding-release-q3", "text": "What retention, hold and disposal rules apply to each evidence artifact?", "kind": "retention", "answer_data": [ "policy_ref", "retention_trigger", "hold_status", "disposition_authority" ] }, { "id": "finding-release-q4", "text": "Which versioned export mapping preserves metric meaning, provenance and visible holds?", "kind": "interoperability", "answer_data": [ "mapping_version", "field_mapping", "loss_report", "consumer_acknowledgement" ] } ], "data_elements": [ { "id": "finding-release-data-release", "name": "Release policy binding", "description": "Role-based authority, field and artifact visibility, license, retention policy and export limitations.", "value_kind": "object", "cardinality": "1", "required": true, "source_refs": [ "SRC-003", "SRC-004", "SRC-007" ] } ], "artifacts": [ { "id": "finding-release-artifact", "name": "Disclosure and retention manifest", "description": "Versioned evidence projection for disclosure and continued availability. Links may replace protected payloads; unknowns and access limits remain explicit.", "media_or_form": [ "application/json", "text/plain" ], "serial": true, "identity_strategy": "Authoritative master-system artifact identifier and revision first; governed IRI second; assigned UUID third. Digest identifies bytes, not reporting authority. Sequence numbers and timestamps are metadata.", "source_refs": [ "SRC-003", "SRC-004", "SRC-007" ] } ], "inline_only_rationale": null } ] } ] } ] }, "functions": [ { "id": "function-register-result", "name": "Register a reported result", "description": "Proposed local operation, not implemented. Create a local result record from an attributable report, preserving unresolved bindings.", "inputs": [ "issuer assertion", "subject and benchmark bindings", "metric outcomes" ], "outputs": [ "result identifier and revision", "missing-evidence report", "refusal with reasons if authority, evidence or concurrency checks fail" ], "preconditions": [ "Actor has a purpose-scoped role and expected record revision", "Reporting authority is established", "A metric has a definition and either a value or an explicit absence state" ], "effects": [ "Create an attributable local record; do not certify the score", "Append an attributable local change record; no external action is authorized by this function" ], "source_refs": [ "SRC-001", "SRC-004" ] }, { "id": "function-bind-evidence", "name": "Bind reproduction evidence", "description": "Proposed local operation, not implemented. Attach versioned references and integrity metadata without executing their contents.", "inputs": [ "result revision", "artifact references and digests", "access classification" ], "outputs": [ "updated evidence index", "unresolved or mismatched reference report", "refusal with reasons if authority, evidence or concurrency checks fail" ], "preconditions": [ "Actor has a purpose-scoped role and expected record revision", "Artifact provenance and permitted access are recorded", "Digest algorithm and byte scope are known when a digest is supplied" ], "effects": [ "Append evidence bindings and availability states; preserve protected payload boundaries", "Append an attributable local change record; no external action is authorized by this function" ], "source_refs": [ "SRC-003", "SRC-007" ] }, { "id": "function-check-result-record", "name": "Check record consistency", "description": "Proposed local operation, not implemented. Check local metadata, reference and declared constraint consistency under an adopted profile.", "inputs": [ "result revision", "pinned profile", "available evidence" ], "outputs": [ "structured consistency report", "unverified checks list", "refusal with reasons if authority, evidence or concurrency checks fail" ], "preconditions": [ "Actor has a purpose-scoped role and expected record revision", "The intended profile and check scope are explicit", "Unavailable evidence is treated as unknown" ], "effects": [ "Record check outcomes; do not run a benchmark or elevate a metadata check to independent verification", "Append an attributable local change record; no external action is authorized by this function" ], "source_refs": [ "SRC-002", "SRC-004", "SRC-006" ] }, { "id": "function-record-verification", "name": "Record a verification assertion", "description": "Proposed local operation, not implemented. Link an authorized reviewer statement to the exact result revision and checked scope.", "inputs": [ "reviewer assertion", "independence disclosure", "criteria and evidence refs" ], "outputs": [ "scoped verification assertion", "conflict or insufficiency report", "refusal with reasons if authority, evidence or concurrency checks fail" ], "preconditions": [ "Actor has a purpose-scoped role and expected record revision", "Reviewer identity and authority are established", "Method, tolerance and scope are explicit; reproduction claims reference an observed attempt" ], "effects": [ "Append the assertion and its limitations; preserve contradictory assertions without silently resolving them", "Append an attributable local change record; no external action is authorized by this function" ], "source_refs": [ "SRC-001", "SRC-004", "SRC-007" ] }, { "id": "function-record-comparison", "name": "Record a comparison assessment", "description": "Proposed local operation, not implemented. Store a supplied baseline comparison with its compatibility limits and derivation.", "inputs": [ "result and baseline revisions", "comparison assessment", "derivation and uncertainty refs" ], "outputs": [ "comparison assertion", "incomparable or unknown status", "refusal with reasons if authority, evidence or concurrency checks fail" ], "preconditions": [ "Actor has a purpose-scoped role and expected record revision", "Baseline is pinned", "Protocol, sample, metric and measurement boundary differences have explicit dispositions" ], "effects": [ "Link the assessment; do not update rankings or approve deployment", "Append an attributable local change record; no external action is authorized by this function" ], "source_refs": [ "SRC-004", "SRC-005", "SRC-006" ] }, { "id": "function-revise-result", "name": "Revise or retire the local result", "description": "Proposed local operation, not implemented. Record an authorized correction or withdrawal with a resolvable predecessor and retention disposition.", "inputs": [ "current revision", "authorized decision", "replacement or withdrawal basis" ], "outputs": [ "new revision or tombstone", "change and notice record", "refusal with reasons if authority, evidence or concurrency checks fail" ], "preconditions": [ "Actor has a purpose-scoped role and expected record revision", "Mutation authority and expected revision match", "Retention holds and disclosure rules are checked" ], "effects": [ "Preserve correction lineage; retire invalid claims; let external custodians execute payload disposal under policy", "Append an attributable local change record; no external action is authorized by this function" ], "source_refs": [ "SRC-001", "SRC-003", "SRC-007" ] } ], "composition": [ { "target": "WM-AI-003", "relation": "REFERENCE", "purpose": "Proposed binding to producing evaluation context. Registry parent does not establish type inheritance or required containment; evaluator retains execution and evaluation lifecycle.", "required": false, "source_refs": [ "SRC-001", "SRC-004" ] }, { "target": "WM-AI-009", "relation": "REFERENCE", "purpose": "Proposed external dataset/benchmark revision binding; curation and protocol lifecycle remain outside this result.", "required": false, "source_refs": [ "SRC-003", "SRC-006" ] }, { "target": "WM-SFT-004", "relation": "REFERENCE", "purpose": "Proposed tested model artifact binding where applicable; no weight, deployment or artifact lifecycle ownership.", "required": false, "source_refs": [ "SRC-001", "SRC-007" ] }, { "target": "W3C PROV-O", "relation": "ALIGN", "purpose": "Conceptual entity, generation, attribution and revision mapping; executable mapping and conformance deferred.", "required": false, "source_refs": [ "SRC-001" ] }, { "target": "W3C DQV and DCAT 3", "relation": "ALIGN", "purpose": "Dataset-quality measurement analogy and evidence distribution metadata; do not assert all AI scores are DQV dataset quality measurements.", "required": false, "source_refs": [ "SRC-002", "SRC-003" ] } ], "serviceLayers": { "dimension": { "owner_package_requirements": [ "Accountable result custodian role and reporting authority", "Master result namespace and identity resolution policy", "Pinned benchmark, evidence access and retention profiles", "Review and independence criteria with escalation route" ], "namespace_guidance": "Use a governed result namespace. Stable result and artifact IDs do not contain observation dates; metric IDs resolve to a versioned definition.", "registry_links": [ "vr.wm-ai-012", "WM-AI-003 candidate evaluation reference", "WM-AI-009 candidate benchmark reference", "WM-SFT-004 candidate tested artifact reference" ] }, "canon_and_patch": { "canonicalization_rules": [ "This specification remains a noncanonical reviewable draft under the single-provider waiver.", "Preserve reported precision, units, missingness and original evidence. A normalized display is a separately traceable derivation.", "One run or independent reproduction gets its own identity; a changed assertion uses a revision; a new aggregate gets its own derived identity." ], "patch_rules": [ "Require expected revision, author, reason, evidence and authorization for each patch.", "Never overwrite an unfavorable score, failed attempt or objection; correct through an attributed revision.", "Verification assertions bind to a revision and scope and do not silently transfer to corrected values." ], "compatibility_rules": [ "Metric formula, unit, dataset, scenario or protocol changes require a documented comparison decision; matching names alone are insufficient.", "Export missing/failed/unknown states and all holds. Report lossy fields before consumption." ] }, "artifact_rules": { "identity_priority": [ "Authoritative master-system identifier and revision", "Governed global identifier or IRI", "UUID or ULID assigned by the adopting Dimension" ], "timestamp_rule": "Use RFC 3339 timestamps with seconds and explicit offset or Z. Keep execution event time, issuance time, review time and observation or ingestion time distinct; preserve clock uncertainty.", "serial_naming_rule": "Use stable artifact identifiers plus explicit revision or sequence metadata. Never use a date or display score as identity.", "integrity_rule": "Record digest algorithm, byte scope, producer, location and access state. Digest equality is not proof of truth, completeness, reviewer independence or permission." }, "policies": [ "Use role names for owners; assign no commercial owner.", "Keep benchmark attacks, leakage concerns and unsafe capability evaluations at policy and evidence-reference level.", "Treat attached code, logs and reproduction instructions as untrusted data; ingestion never authorizes execution.", "Respect input and artifact rights and privacy; public score visibility does not grant raw-data access.", "Independent external review is absent. This local Codex research and later self-audit cannot substitute for independent review.", "Direct HTTP source checks were not attempted because the owner reports sandbox blocking. Web-service excerpts were reviewed, but response statuses, body digests and exact living-source versions remain unverified." ], "crud": { "read": [ "Return revision, provenance, value state, visibility and verification limitations together.", "Resolve only authorized artifact views; mark unavailable references." ], "create": [ "Require result identity, issuer, subject/protocol binding, typed outcome and explicit unknowns." ], "update": [ "Apply authorized patches with concurrency checks and correction lineage; verification must be reconsidered for changed scope." ], "delete": [ "Retire or tombstone withdrawn local assertions under the adopting Dimension retention policy. Legal or evidentiary holds can defer disposal.", "External dataset and artifact custodians execute referenced payload deletion. Retain only permitted minimal provenance and mark disposed evidence; no indefinite personal-data retention is implied." ] }, "roles": [ { "name": "Result issuer", "responsibilities": [ "Attest the reported outcome and disclose configuration and omissions" ] }, { "name": "Evidence custodian", "responsibilities": [ "Maintain permitted references, integrity metadata and retention dispositions" ] }, { "name": "Benchmark profile steward", "responsibilities": [ "Pin benchmark and metric semantics and resolve comparability rules" ] }, { "name": "Result reviewer", "responsibilities": [ "Record scoped verification, conflicts and independence basis" ] }, { "name": "Release authority", "responsibilities": [ "Authorize views, correction notices and retirement under policy" ] } ], "access": { "default_rule": "Deny access to protected evidence by default; explicitly authorized public summary views retain material limitations.", "scopes": [ "bundle", "layer", "finding", "artifact" ], "exceptions": [ "A documented policy can permit scoped reviewer access without public release.", "Emergency or mandatory disclosure needs recorded authority, scope and later review; it does not imply access to all artifacts." ], "audit_requirements": [ "Record actor, purpose, affected revision, changed fields, access decision and evidence references.", "Use an external audit-log master for execution and retention of audit trails; this model retains links." ] }, "agents_bootstrap": { "filename": "AGENTS.md", "required_fields": [ "Name", "Type", "Specification URL", "Storage type URL", "Interface URL", "Processes URL" ], "read_order": [ "AGENTS.md", "spec.yaml", "Owner and Dimension policies", "Pinned benchmark profile and external master references", "Result revision, evidence availability, verification assertions and holds" ] } }, "coverage": { "claim": "Source-grounded proposed structure for a persistent benchmark result record with qualified measurements, reproduction evidence and verification assertions. Separate frozen local no-tools self-audit completed. Independent review, source/version checks, benchmark-profile adoption and executable conformance remain holds; this is a noncanonical reviewable draft, not a verified benchmark result or deployment approval.", "confidence": "medium", "checklist": [ { "dimension": "identity", "status": "covered", "notes": "Result, run, revision, aggregate and artifact identities are distinct." }, { "dimension": "lifecycle", "status": "covered", "notes": "Local correction, supersession, withdrawal and retirement retain their basis." }, { "dimension": "relationships", "status": "covered", "notes": "Evaluation, benchmark/dataset and tested artifact are external candidate references." }, { "dimension": "temporal", "status": "covered", "notes": "Execution, issuance, review and ingestion times remain separate." }, { "dimension": "provenance", "status": "covered", "notes": "Issuer, transformations, scoring lineage and evidence digests are represented." }, { "dimension": "ownership", "status": "covered", "notes": "Custodian, reporting, review and release roles are distinct." }, { "dimension": "validation", "status": "covered", "notes": "Metadata checks, protocol checks and reproduction assertions have different scopes." }, { "dimension": "access", "status": "covered", "notes": "Restricted artifacts and public summaries receive different views." }, { "dimension": "retention and deletion", "status": "covered", "notes": "Policy-bound retirement and external payload disposal preserve permitted provenance." }, { "dimension": "interoperability", "status": "gap", "notes": "Conceptual PROV-O, DQV and DCAT mappings need executable schemas and fixtures." }, { "dimension": "direct properties", "status": "covered", "notes": "Metric values, units, validity states, counts and uncertainty describe this informational entity." }, { "dimension": "recognition and observation", "status": "covered", "notes": "Issuer assertion, execution references and evidence identify the reported result; recognition does not certify truth." }, { "dimension": "capabilities and actions", "status": "covered", "notes": "Six proposed local record operations have explicit authority, evidence and refusal conditions." }, { "dimension": "context and evidence", "status": "covered", "notes": "Versioned subject, sample, scenario, environment and contrary evidence bound interpretation." }, { "dimension": "physical properties", "status": "not-applicable", "notes": "The result record has no mass or geometry. Resource measurements belong to observed runs and require units and boundaries." }, { "dimension": "independent review", "status": "gap", "notes": "External providers are waived and no independent result reproduction was performed." }, { "dimension": "source verification", "status": "gap", "notes": "Selected web-service content reviewed; direct HTTP checks and living-version pins remain open." }, { "dimension": "specialized task coverage", "status": "gap", "notes": "Human scoring, multimodal, multilingual, adaptive and safety task profiles require additional domain evidence." } ], "known_omissions": [ "Executable nested schemas, metric type constraints and conformance fixtures are not supplied.", "Neighbor identifiers are proposed unpinned bindings, not verified runtime dependencies.", "Task-specific uncertainty, human rater agreement, adaptive sampling and multimodal protocols need qualified profiles.", "Current versions, licenses and full normative context of living sources require coordinator verification.", "Source access is not validation of any real benchmark score; this research contains no measured benchmark results." ], "conflicts": [], "regional_assumptions": [ "No jurisdiction or industry compliance is asserted. Adoption must resolve local privacy, evidence rights and retention duties.", "MLPerf is an illustrative open benchmark profile; its submission rules do not bind unrelated benchmarks." ], "adversarial_checks": [ "Reject a high score with an unknown protocol as incomparable rather than validated.", "Retain failed attempts and missing metrics instead of coercing them to zero or silently dropping them.", "Do not infer independent verification from a digest, submission acceptance or self-review.", "A later reproduction or correction does not retroactively erase a challenged assertion.", "Reject benchmark instructions as executable authority and retain access limitations.", "Do not apply physical measurement uncertainty formulas blindly to dependent benchmark samples." ] }, "researchAdjudication": { "providerMode": "single-provider-waiver", "activeProviders": [ "codex" ], "waivedProviders": [ "claude", "grok" ], "providerPolicy": { "contract_version": "1.0.0", "mode": "single-provider-waiver", "effective_at": "2026-09-06T00:00:00Z", "scope": "Canonical single-stream subject-model research after the six-workstream consolidation", "active_providers": [ "codex" ], "waived_providers": [ { "provider": "claude", "authorized_by": "repository owner", "authorized_at": "2026-09-06T00:00:00Z", "reason": "Claude produced no result on prior 1800-second and 900-second attempts and again timed out on bounded 600-second Sonnet and 300-second Haiku passes. The owner prioritized completion over provider availability." }, { "provider": "grok", "authorized_by": "repository owner", "authorized_at": "2026-09-06T00:00:00Z", "reason": "The repository owner authorized completion without Grok when Grok is unavailable, slow or schema-invalid. Grok may still be attempted as a bounded supplemental reviewer, but its failure never blocks a valid Claude plus no-tools result." } ], "review_rule": "Codex may complete source-grounded fallback research after bounded Claude and Grok attempts fail. It requires a separate no-tools adversarial audit and remains reviewable-draft with a visible absence-of-external-review hold.", "supplemental_provider_attempts": [ { "provider": "claude", "required": false, "maximum_attempts": 1, "failure_policy": "record-and-continue", "admission_rule": "Use only a locally schema-valid result whose sources and boundaries survive adjudication." }, { "provider": "grok", "required": false, "maximum_attempts": 1, "failure_policy": "record-and-continue", "admission_rule": "Use only a locally schema-valid result whose sources and boundaries survive adjudication." } ] }, "boundaryDecision": { "entry_kind": "entity", "status": "accepted", "rationale": "A result is an attributable persistent informational entity, distinct from the producing evaluation event. Metric vectors and explicit aggregates can be contents of that entity without making every result a process or aggregate master. Separate runs and reproductions retain separate identities; corrections use revisions. The registry standalone-mm classification and parent link do not establish subject-kind inheritance." }, "decisions": [ { "concept": "Result identity and boundary", "disposition": "accepted", "rationale": "The claim, issuer, master identifier and result kind provide a stable root. Run identity, reporting time and display score cannot substitute for result identity." }, { "concept": "Evaluation parent and external masters", "disposition": "qualified", "rationale": "No direct relation ledger row was present. The registry parent is reconciled as an evaluation-context reference, while dataset and model artifact references remain proposed and unpinned. No evaluator execution, dataset curation or model lifecycle is owned locally." }, { "concept": "Subject and protocol pinning", "disposition": "accepted with unknown state", "rationale": "Configuration, transformations, benchmark release, scenario and deviations are explicit. A missing revision can be recorded, but cannot establish comparability or verified identity." }, { "concept": "Sample accounting and contamination", "disposition": "accepted", "rationale": "Eligible, attempted, completed and excluded cases are distinguished. Overlap assessment requires evidence and can remain unknown; no absence-of-contamination claim is inferred from missing evidence." }, { "concept": "Execution and reproduction boundary", "disposition": "accepted", "rationale": "Observed run conditions and reproduction artifacts are reference data. The functions do not schedule or execute tests. Instructions embedded in evidence remain untrusted and need separate authority before any future execution." }, { "concept": "Metric values and missing outcomes", "disposition": "accepted", "rationale": "Metric definition, unit, direction, precision, slice and value status accompany each outcome. Failed, censored, missing and nonapplicable values are not numeric zero. A failed result can contain a typed absence outcome." }, { "concept": "Aggregation and uncertainty", "disposition": "qualified", "rationale": "The proposed account preserves included and failed attempts, weighting, denominators and uncertainty assumptions. The NIST measurement analogy does not supply a universal statistical method for dependent benchmark samples; task-specific statistical validation remains deferred." }, { "concept": "Verification independence", "disposition": "accepted", "rationale": "Local checking, review and reproduction are separate scoped assertions tied to a result revision. Reviewer relationships, criteria and evidence support any independence claim; a digest or submission acceptance cannot establish it." }, { "concept": "Challenge and correction", "disposition": "accepted", "rationale": "Contrary evidence and resolutions remain linked to the challenged revision. Corrections, withdrawal and later reproduction have separate meanings, and verification does not silently transfer to a revised score." }, { "concept": "Comparison and generalization", "disposition": "limited", "rationale": "Baseline revisions and compatibility differences are required context for any comparison assertion. No leaderboard operation, general capability certification or deployment approval follows from recording a comparison." }, { "concept": "Release, access and retention", "disposition": "accepted with policy dependency", "rationale": "Public summary access does not grant raw evidence access. Retention and minimal tombstones depend on the adopting policy, and external custodians execute payload disposal. Immutable lineage does not require indefinite protected-payload retention." }, { "concept": "Functions and service layers", "disposition": "accepted as proposed", "rationale": "All six local operations have evidence and authority preconditions, explicit refusal and limited record effects. All eight service sections preserve role-based ownership, revision concurrency, artifact identity, temporal distinctions and external audit ownership." }, { "concept": "Source applicability and interoperability", "disposition": "deferred for operational use", "rationale": "DQV is a dataset-quality analogy, DCAT is evidence metadata and MLPerf is one illustrative benchmark profile. Mutable source versions, nested instance schemas, mappings and executable acceptance fixtures are unresolved rather than silently claimed conformant." }, { "concept": "Independent research review and source checks", "disposition": "waived and held", "rationale": "Claude and Grok were skipped with zero attempts. This separate Codex self-audit adds no facts and is not independent review. Selected web-service text was available for seven sources, but direct HTTP was not attempted and all local response statuses remain unmeasured." } ], "publicationHolds": [ "Independent external review is absent under the owner-authorized single-provider waiver. Claude and Grok were skipped with zero attempts; the separate frozen local Codex no-tools self-audit is not a second-provider review.", "Source and version verification remains incomplete. Seven sources supplied selected readable web-service text; direct HTTP checks were not attempted under the owner-reported sandbox restriction. Status codes and body hashes are unmeasured, and mutable documentation and policy revisions are not pinned. The coordinator must run check_sources.py and assess substance, versions and rights separately.", "Benchmark-specific acceptance, statistical uncertainty, contamination assessment, human-rating and specialized task profiles, reviewer independence criteria, privacy and retention require qualified adoption review. No measured score, benchmark certification, general capability or deployment fitness is established.", "Nested instance schemas, pinned neighbor bindings, PROV-O/DQV/DCAT mappings and executable conformance fixtures remain incomplete. Proposed local functions are not implemented and no runtime conformance is claimed.", "Independent external review was explicitly waived by the repository owner; this codex-only result remains a reviewable draft." ], "deferredResearch": [ "Restore independent external review before canonical or publishable-draft promotion.", "Verify source response bodies, exact editions or commit pins, applicable licenses and complete normative context outside the restricted sandbox.", "Develop task-specific profiles and adversarial fixtures for missing scores, failed attempts, biased selection, incompatible comparisons, disputed review and correction lineage." ] }, "statistics": { "sources": 7, "bundles": 6, "layers": 12, "findings": 12, "questions": 48, "artifacts": 12, "functions": 6 } }