{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 5.65, "prompt_chars": 11085, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No definition of 'active customer' provided in data.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No definition of 'active customer' provided in data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 1, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 8.06, "prompt_chars": 11085, "raw": "{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Counted customers with contract_status='open' in customers table.\"}\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted customers with contract_status='open' in customers table."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 11.71, "prompt_chars": 11085, "raw": "{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Counted customers with contract_status='open'; status has no timestamp field.\"}\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted customers with contract_status='open'; status has no timestamp field."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 15.82, "prompt_chars": 11102, "raw": "{\"answer\": 46493, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum invoice amounts minus returns dated within Q1 2026 (by invoice/return date).\"}\n", "obj": {"answer": 46493, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum invoice amounts minus returns dated within Q1 2026 (by invoice/return date)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 20.9, "prompt_chars": 11102, "raw": "{\"answer\": 134719, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum of invoice amounts minus returns dated within period (no catalogue definition).\"}\n", "obj": {"answer": 134719, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of invoice amounts minus returns dated within period (no catalogue definition)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 11.96, "prompt_chars": 11082, "raw": "{\"answer\": 40, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date present and <= 2026-05-31.\"}\n", "obj": {"answer": 40, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date present and <= 2026-05-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 20.9, "prompt_chars": 11082, "raw": "{\"answer\": 53, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = shipments with customer_confirm_date on/before 2026-08-31.\"}\n", "obj": {"answer": 53, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = shipments with customer_confirm_date on/before 2026-08-31."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 20.51, "prompt_chars": 11082, "raw": "{\"answer\": 47, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date set and <= 2026-06-30.\"}\n", "obj": {"answer": 47, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date set and <= 2026-06-30."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 34.13, "prompt_chars": 11102, "raw": "{\"answer\": 27120, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Net = sum(gross-discount) for orders in period minus returns dated by 2026-08-31.\"}\n", "obj": {"answer": 27120, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Net = sum(gross-discount) for orders in period minus returns dated by 2026-08-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 8.42, "prompt_chars": 11085, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No definition of 'active customer' or time-varying status provided.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No definition of 'active customer' or time-varying status provided."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 1, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 9.82, "prompt_chars": 11085, "raw": "{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active = contract_status='open'; no other definition available in data.\"}\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = contract_status='open'; no other definition available in data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 5.31, "prompt_chars": 11085, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No 'active customer' definition provided in data or context.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No 'active customer' definition provided in data or context."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 1, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 19.24, "prompt_chars": 11136, "raw": "{\"answer\": 100881, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Order value = gross - discount + freight + tax, summed for non-cancelled orders in range.\"}\n", "obj": {"answer": 100881, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Order value = gross - discount + freight + tax, summed for non-cancelled orders in range."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 9.51, "prompt_chars": 11085, "raw": "{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Counted customers with contract_status='open' as active.\"}\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted customers with contract_status='open' as active."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 11.29, "prompt_chars": 11085, "raw": "{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active = contract_status='open' as of 2026-03-31; 20 of 24 customers.\"}\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = contract_status='open' as of 2026-03-31; 20 of 24 customers."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 14.4, "prompt_chars": 11085, "raw": "{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active = customers.contract_status='open'; no historical status dates available.\"}\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = customers.contract_status='open'; no historical status dates available."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 44.34, "prompt_chars": 11090, "raw": "{\"answer\": 22, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=customer_confirm_date≤7/31/26; on time if ≤(revised or promised)_date.\"}\n", "obj": {"answer": 22, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=customer_confirm_date≤7/31/26; on time if ≤(revised or promised)_date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 26.4, "prompt_chars": 11090, "raw": "{\"answer\": 13, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date ≤2026-01-31; on time if ≤(revised_promised_date or promised_date).\"}\n", "obj": {"answer": 13, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date ≤2026-01-31; on time if ≤(revised_promised_date or promised_date)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 28.94, "prompt_chars": 11090, "raw": "{\"answer\": 12, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"On-time = confirm_date ≤ revised (or promised) date, delivered by 2026-01-15.\"}\n", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "On-time = confirm_date ≤ revised (or promised) date, delivered by 2026-01-15."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 67.02, "prompt_chars": 11090, "raw": "{\"answer\": 23, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=confirm_date≤2026-08-31; on-time if confirm≤(revised or) promised_date.\"}\n", "obj": {"answer": 23, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=confirm_date≤2026-08-31; on-time if confirm≤(revised or) promised_date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 43.37, "prompt_chars": 11090, "raw": "{\"answer\": 11, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=confirm_date≤2025-12-31; on-time=confirm_date≤(revised or promised_date).\"}\n", "obj": {"answer": 11, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=confirm_date≤2025-12-31; on-time=confirm_date≤(revised or promised_date)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 24.36, "prompt_chars": 11090, "raw": "{\"answer\": 13, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"On-time = customer_confirm_date <= revised (else) promised_date, confirmed by 2026-02-01.\"}\n", "obj": {"answer": 13, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "On-time = customer_confirm_date <= revised (else) promised_date, confirmed by 2026-02-01."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 16.21, "prompt_chars": 11148, "raw": "{\"answer\": 27, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date set and on/before 2026-07-31, supplier HAL.\"}\n", "obj": {"answer": 27, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date set and on/before 2026-07-31, supplier HAL."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 34.84, "prompt_chars": 11151, "raw": "{\"answer\": 13, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date set, ≤2026-08-31, for HAL orders under C-118.\"}\n", "obj": {"answer": 13, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date set, ≤2026-08-31, for HAL orders under C-118."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 34.51, "prompt_chars": 11151, "raw": "{\"answer\": 11, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date present & ≤2026-06-30, HAL+C-118 orders.\"}\n", "obj": {"answer": 11, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date present & ≤2026-06-30, HAL+C-118 orders."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 28.53, "prompt_chars": 11187, "raw": "{\"answer\": 40952, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum(gross-discount+freight+tax) for HAL non-cancelled orders, order_date May-Jul 2026.\"}\n", "obj": {"answer": 40952, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum(gross-discount+freight+tax) for HAL non-cancelled orders, order_date May-Jul 2026."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 18.92, "prompt_chars": 11202, "raw": "{\"answer\": 11, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Weekdays from 2026-03-26 (excl) to 2026-04-11 (incl) for O023.\"}\n", "obj": {"answer": 11, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Weekdays from 2026-03-26 (excl) to 2026-04-11 (incl) for O023."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 42.38, "prompt_chars": 11140, "raw": "{\"answer\": 6, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"On-time = confirm date ≤ (revised or original) promised date.\"}\n", "obj": {"answer": 6, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "On-time = confirm date ≤ (revised or original) promised date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 42.35, "prompt_chars": 11140, "raw": "{\"answer\": 5, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=customer_confirm_date≤2026-06-30; on-time vs promised/revised date.\"}\n", "obj": {"answer": 5, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=customer_confirm_date≤2026-06-30; on-time vs promised/revised date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 31.28, "prompt_chars": 11187, "raw": "{\"answer\": 87358, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum of gross-discount+freight+tax for non-cancelled HAL orders, Jan-Mar 2026.\"}\n", "obj": {"answer": 87358, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of gross-discount+freight+tax for non-cancelled HAL orders, Jan-Mar 2026."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 6.57, "prompt_chars": 11092, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No churn date field exists; contract_status has no timestamp.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No churn date field exists; contract_status has no timestamp."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 21.47, "prompt_chars": 11202, "raw": "{\"answer\": 11, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Business days (Mon-Fri) from 2026-03-12 to 2026-03-27, order O025/shipment S022.\"}\n", "obj": {"answer": 11, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Business days (Mon-Fri) from 2026-03-12 to 2026-03-27, order O025/shipment S022."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 9.48, "prompt_chars": 11071, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No cost/COGS data provided; gross margin cannot be computed.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No cost/COGS data provided; gross margin cannot be computed."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "schema", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 5.42, "prompt_chars": 11086, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No damage/transit-condition field exists in shipments data.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damage/transit-condition field exists in shipments data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "schema", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 5.79, "prompt_chars": 11098, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No customer satisfaction score data provided in tables.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score data provided in tables."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "schema", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 4.96, "prompt_chars": 11140, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No 'customer's own working calendar' or business-day/holiday definition provided.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No 'customer's own working calendar' or business-day/holiday definition provided."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 23.22, "prompt_chars": 11160, "raw": "{\"answer\": 44500, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum of gross_amount for ZEN orders, non-cancelled, dated 2026-06-01 to 2026-08-31.\"}\n", "obj": {"answer": 44500, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of gross_amount for ZEN orders, non-cancelled, dated 2026-06-01 to 2026-08-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 7.16, "prompt_chars": 11088, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No definition of 'active' or future status data available.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No definition of 'active' or future status data available."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 32.11, "prompt_chars": 11114, "raw": "{\"answer\": 28, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = shipment has customer_confirm_date on/before 2026-08-31.\"}\n", "obj": {"answer": 28, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = shipment has customer_confirm_date on/before 2026-08-31."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "schema", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 70.62, "prompt_chars": 11140, "raw": "{\"answer\": 6, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"On-time = confirm_date ≤ revised/promised date, delivered ≤2026-07-31, HAL C-118 orders.\"}\n", "obj": {"answer": 6, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "On-time = confirm_date ≤ revised/promised date, delivered ≤2026-07-31, HAL C-118 orders."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 19.97, "prompt_chars": 12478, "raw": "{\"answer\": 21, \"definition_id\": \"active_customer_past_year\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Customers with ≥1 order between 2025-09-01 and 2026-08-31.\"}\n", "obj": {"answer": 21, "definition_id": "active_customer_past_year", "definition_version": 1, "unanswerable": false, "note": "Customers with ≥1 order between 2025-09-01 and 2026-08-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 17.35, "prompt_chars": 12478, "raw": "{\"answer\": 23, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Customers with an order in the trailing 12 months (2025-06-01 to 2026-06-01).\"}\n", "obj": {"answer": 23, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with an order in the trailing 12 months (2025-06-01 to 2026-06-01)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 20.36, "prompt_chars": 12478, "raw": "{\"answer\": 23, \"definition_id\": \"active_customer_ordered_past_year\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Customers with an order dated 2025-07-15 to 2026-07-15.\"}\n", "obj": {"answer": 23, "definition_id": "active_customer_ordered_past_year", "definition_version": 1, "unanswerable": false, "note": "Customers with an order dated 2025-07-15 to 2026-07-15."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 23.97, "prompt_chars": 12495, "raw": "{\"answer\": 27120, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum(gross-discount) for orders dated Jul1-Aug31, minus returns dated same period.\"}\n", "obj": {"answer": 27120, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum(gross-discount) for orders dated Jul1-Aug31, minus returns dated same period."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 32.26, "prompt_chars": 12495, "raw": "{\"answer\": 108960, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Net = (gross-discount) for non-cancelled orders by order_date, minus returns.\"}\n", "obj": {"answer": 108960, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Net = (gross-discount) for non-cancelled orders by order_date, minus returns."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 20.29, "prompt_chars": 12475, "raw": "{\"answer\": 42, \"definition_id\": \"delivered_customer_confirm_date\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date present, on/before 2026-05-31.\"}\n", "obj": {"answer": 42, "definition_id": "delivered_customer_confirm_date", "definition_version": 1, "unanswerable": false, "note": "Delivered = customer_confirm_date present, on/before 2026-05-31."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 22.22, "prompt_chars": 12475, "raw": "{\"answer\": 53, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Shipments with customer_confirm_date on/before 2026-08-31 (delivery = customer receipt).\"}\n", "obj": {"answer": 53, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with customer_confirm_date on/before 2026-08-31 (delivery = customer receipt)."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 37.88, "prompt_chars": 12495, "raw": "{\"answer\": 93800, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum(gross-discount) for non-cancelled orders in Q2, minus matched returns.\"}\n", "obj": {"answer": 93800, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum(gross-discount) for non-cancelled orders in Q2, minus matched returns."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 40.97, "prompt_chars": 12475, "raw": "{\"answer\": 47, \"definition_id\": \"delivered_customer_confirm_date\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date set and ≤ 2026-06-30.\"}\n", "obj": {"answer": 47, "definition_id": "delivered_customer_confirm_date", "definition_version": 1, "unanswerable": false, "note": "Delivered = customer_confirm_date set and ≤ 2026-06-30."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 21.2, "prompt_chars": 12478, "raw": "{\"answer\": 22, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Used 'ordered in past year' definition (past 12 months of order_date).\"}\n", "obj": {"answer": 22, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Used 'ordered in past year' definition (past 12 months of order_date)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 30.57, "prompt_chars": 12478, "raw": "{\"answer\": 22, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active = ordered in past 12 months (2025-02-01 to 2026-01-31).\"}\n", "obj": {"answer": 22, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = ordered in past 12 months (2025-02-01 to 2026-01-31)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 32.14, "prompt_chars": 12529, "raw": "{\"answer\": 100881, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum of (gross−discount+freight+tax) for non-cancelled orders dated 2026‑05‑01–07‑31.\"}\n", "obj": {"answer": 100881, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of (gross−discount+freight+tax) for non-cancelled orders dated 2026‑05‑01–07‑31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 37.04, "prompt_chars": 12483, "raw": "{\"answer\": 22, \"definition_id\": \"delivery=customer_confirm_date; on_time=confirm<=revised_or_promised_date\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Used customer confirm as delivery; compared to revised/promised date.\"}\n", "obj": {"answer": 22, "definition_id": "delivery=customer_confirm_date; on_time=confirm<=revised_or_promised_date", "definition_version": 1, "unanswerable": false, "note": "Used customer confirm as delivery; compared to revised/promised date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 17.67, "prompt_chars": 12478, "raw": "{\"answer\": 22, \"definition_id\": \"ordered_in_past_year\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Distinct customers with orders 2025-03-15 to 2026-03-15.\"}\n", "obj": {"answer": 22, "definition_id": "ordered_in_past_year", "definition_version": 1, "unanswerable": false, "note": "Distinct customers with orders 2025-03-15 to 2026-03-15."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 18.12, "prompt_chars": 12478, "raw": "{\"answer\": 22, \"definition_id\": \"ordered_past_12_months\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Customers with an order dated 2025-02-15 to 2026-02-15.\"}\n", "obj": {"answer": 22, "definition_id": "ordered_past_12_months", "definition_version": 1, "unanswerable": false, "note": "Customers with an order dated 2025-02-15 to 2026-02-15."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 24.29, "prompt_chars": 12483, "raw": "{\"answer\": 11, \"definition_id\": \"delivered=customer_confirm_date; on_time=confirm<=revised_promised_date or promised_date\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Counted shipments confirmed by customer ≤2025-12-31 meeting agreed date.\"}\n", "obj": {"answer": 11, "definition_id": "delivered=customer_confirm_date; on_time=confirm<=revised_promised_date or promised_date", "definition_version": 1, "unanswerable": false, "note": "Counted shipments confirmed by customer ≤2025-12-31 meeting agreed date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 19.52, "prompt_chars": 12478, "raw": "{\"answer\": 23, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active = ordered within prior 12 months (2025-04-01 to 2026-04-01); all but C019.\"}\n", "obj": {"answer": 23, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = ordered within prior 12 months (2025-04-01 to 2026-04-01); all but C019."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 31.49, "prompt_chars": 12483, "raw": "{\"answer\": 13, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=customer_confirm_date≤2026-01-31; on-time vs revised/promised date.\"}\n", "obj": {"answer": 13, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=customer_confirm_date≤2026-01-31; on-time vs revised/promised date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 33.99, "prompt_chars": 12478, "raw": "{\"answer\": 23, \"definition_id\": \"active_customer_past_year\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Applied 'ordered in past year' definition; only C019 had no order.\"}\n", "obj": {"answer": 23, "definition_id": "active_customer_past_year", "definition_version": 1, "unanswerable": false, "note": "Applied 'ordered in past year' definition; only C019 had no order."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 37.21, "prompt_chars": 12483, "raw": "{\"answer\": 12, \"definition_id\": \"on_time_delivery_revised_date\", \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=confirm date ≤2026-01-15; on-time vs revised/promised date.\"}\n", "obj": {"answer": 12, "definition_id": "on_time_delivery_revised_date", "definition_version": null, "unanswerable": false, "note": "Delivered=confirm date ≤2026-01-15; on-time vs revised/promised date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 27.87, "prompt_chars": 12544, "raw": "{\"answer\": 13, \"definition_id\": \"delivered=customer_confirm_date (HAL orders under C-118)\", \"definition_version\": null, \"unanswerable\": false, \"note\": \"Halden shipments, contract C-118, customer_confirm_date ≤ 2026-08-31.\"}\n", "obj": {"answer": 13, "definition_id": "delivered=customer_confirm_date (HAL orders under C-118)", "definition_version": null, "unanswerable": false, "note": "Halden shipments, contract C-118, customer_confirm_date ≤ 2026-08-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 29.83, "prompt_chars": 12483, "raw": "{\"answer\": 13, \"definition_id\": \"delivered=customer_confirm_date; on_time=confirm<=revised_or_promised_date\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"13 of 31 confirmed-delivered shipments by 2026-02-01 met promised/revised date.\"}\n", "obj": {"answer": 13, "definition_id": "delivered=customer_confirm_date; on_time=confirm<=revised_or_promised_date", "definition_version": 1, "unanswerable": false, "note": "13 of 31 confirmed-delivered shipments by 2026-02-01 met promised/revised date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 30.3, "prompt_chars": 12541, "raw": "{\"answer\": 32, \"definition_id\": \"delivery.carrier_scan_reporting_to_supplier\", \"definition_version\": null, \"unanswerable\": false, \"note\": \"Used carrier_scan_date (Halden's reporting basis) ≤ 2026-07-31.\"}\n", "obj": {"answer": 32, "definition_id": "delivery.carrier_scan_reporting_to_supplier", "definition_version": null, "unanswerable": false, "note": "Used carrier_scan_date (Halden's reporting basis) ≤ 2026-07-31."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 38.18, "prompt_chars": 12533, "raw": "{\"answer\": 6, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=customer_confirm_date; on-time vs promised/revised_promised_date, HAL C-118 orders.\"}\n", "obj": {"answer": 6, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=customer_confirm_date; on-time vs promised/revised_promised_date, HAL C-118 orders."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 23.88, "prompt_chars": 12580, "raw": "{\"answer\": 40952, \"definition_id\": \"invoiceable_amount_gross_minus_discount_plus_freight_plus_tax\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum of HAL orders (O006,O015,O029,O060,O061) net+freight+tax, excl. cancelled.\"}\n", "obj": {"answer": 40952, "definition_id": "invoiceable_amount_gross_minus_discount_plus_freight_plus_tax", "definition_version": 1, "unanswerable": false, "note": "Sum of HAL orders (O006,O015,O029,O060,O061) net+freight+tax, excl. cancelled."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 35.58, "prompt_chars": 12533, "raw": "{\"answer\": 6, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=customer_confirm_date≤6/30; on-time vs revised/promised date, HAL+C-118 orders.\"}\n", "obj": {"answer": 6, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=customer_confirm_date≤6/30; on-time vs revised/promised date, HAL+C-118 orders."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 51.24, "prompt_chars": 12533, "raw": "{\"answer\": 6, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=customer_confirm_date≤7/31/26; on-time vs revised/promised date.\"}\n", "obj": {"answer": 6, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=customer_confirm_date≤7/31/26; on-time vs revised/promised date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 77.08, "prompt_chars": 12544, "raw": "{\"answer\": 16, \"definition_id\": \"carrier_scan_delivery\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Counted HAL/C-118 shipments with carrier_scan_date ≤ 2026-06-30 (logistics-partner delivery view).\"}\n", "obj": {"answer": 16, "definition_id": "carrier_scan_delivery", "definition_version": 1, "unanswerable": false, "note": "Counted HAL/C-118 shipments with carrier_scan_date ≤ 2026-06-30 (logistics-partner delivery view)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 20.93, "prompt_chars": 12595, "raw": "{\"answer\": 11, \"definition_id\": \"business_days_carrier_scan\", \"definition_version\": null, \"unanswerable\": false, \"note\": \"Weekdays from 2026-03-13 to 2026-03-27 (carrier scan date used per logistics note).\"}\n", "obj": {"answer": 11, "definition_id": "business_days_carrier_scan", "definition_version": null, "unanswerable": false, "note": "Weekdays from 2026-03-13 to 2026-03-27 (carrier scan date used per logistics note)."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 27.86, "prompt_chars": 12595, "raw": "{\"answer\": 11, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Weekdays from 2026-03-27 to 2026-04-11 (Mon–Fri only, no holidays).\"}\n", "obj": {"answer": 11, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Weekdays from 2026-03-27 to 2026-04-11 (Mon–Fri only, no holidays)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 34.75, "prompt_chars": 12580, "raw": "{\"answer\": 87358, \"definition_id\": \"invoiceable_value_gross_minus_discount_plus_freight_plus_tax\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum of (gross-discount+freight+tax) for non-cancelled HAL orders, Jan–Mar 2026.\"}\n", "obj": {"answer": 87358, "definition_id": "invoiceable_value_gross_minus_discount_plus_freight_plus_tax", "definition_version": 1, "unanswerable": false, "note": "Sum of (gross-discount+freight+tax) for non-cancelled HAL orders, Jan–Mar 2026."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 7.97, "prompt_chars": 12485, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No churn event dates available; contract_status has no change timestamp.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No churn event dates available; contract_status has no change timestamp."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "prose", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 6.14, "prompt_chars": 12464, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No cost/COGS data provided; cannot compute gross margin.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No cost/COGS data provided; cannot compute gross margin."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "prose", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 6.38, "prompt_chars": 12479, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No damage/condition data tracked in shipments table.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damage/condition data tracked in shipments table."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "prose", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 5.4, "prompt_chars": 12491, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No customer satisfaction score data present in tables.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score data present in tables."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "prose", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 8.44, "prompt_chars": 12533, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No customer-specific holiday calendars provided to compute business days.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer-specific holiday calendars provided to compute business days."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "prose", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 18.17, "prompt_chars": 12481, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"Future date; orders after 2026-09-06 unknown, cannot compute past-year activity.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Future date; orders after 2026-09-06 unknown, cannot compute past-year activity."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 38.94, "prompt_chars": 12553, "raw": "{\"answer\": 44500, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum of gross_amount for ZEN non-cancelled orders dated Jun–Aug 2026.\"}\n", "obj": {"answer": 44500, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of gross_amount for ZEN non-cancelled orders dated Jun–Aug 2026."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 42.23, "prompt_chars": 12507, "raw": "{\"answer\": 28, \"definition_id\": \"delivery_customer_confirm\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date set, on/before 2026-08-31, order under C-204.\"}\n", "obj": {"answer": 28, "definition_id": "delivery_customer_confirm", "definition_version": 1, "unanswerable": false, "note": "Delivered = customer_confirm_date set, on/before 2026-08-31, order under C-204."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 22.99, "prompt_chars": 14572, "raw": "{\"answer\": 16, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Customers with non-cancelled order dated (2026-01-16, 2026-07-15].\"}\n", "obj": {"answer": 16, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Customers with non-cancelled order dated (2026-01-16, 2026-07-15]."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 34.4, "prompt_chars": 14572, "raw": "{\"answer\": 14, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Customers with non-cancelled orders in (2026-03-04, 2026-08-31], 180-day window.\"}\n", "obj": {"answer": 14, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Customers with non-cancelled orders in (2026-03-04, 2026-08-31], 180-day window."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 17.57, "prompt_chars": 14589, "raw": "{\"answer\": 109940, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum(gross-discount) for invoices Apr–Jun 2026 minus returns (R010) same period.\"}\n", "obj": {"answer": 109940, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum(gross-discount) for invoices Apr–Jun 2026 minus returns (R010) same period."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 30.87, "prompt_chars": 14572, "raw": "{\"answer\": 15, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Window 180d (2025-12-03,2026-06-01], non-cancelled orders; 15 customers qualify.\"}\n", "obj": {"answer": 15, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window 180d (2025-12-03,2026-06-01], non-cancelled orders; 15 customers qualify."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 21.79, "prompt_chars": 14589, "raw": "{\"answer\": 33020, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum invoice order values (gross-discount) minus returns dated Jul-Aug 2026.\"}\n", "obj": {"answer": 33020, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum invoice order values (gross-discount) minus returns dated Jul-Aug 2026."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 32.06, "prompt_chars": 14589, "raw": "{\"answer\": 37422, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum (gross-discount) for 7 invoices Q1'26 minus 2 returns (3748).\"}\n", "obj": {"answer": 37422, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum (gross-discount) for 7 invoices Q1'26 minus 2 returns (3748)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 26.94, "prompt_chars": 14569, "raw": "{\"answer\": 52, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Shipments with customer_confirm_date present, on/before 2026-08-31.\"}\n", "obj": {"answer": 52, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present, on/before 2026-08-31."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 18.83, "prompt_chars": 14569, "raw": "{\"answer\": 42, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Counted shipments with customer_confirm_date present and ≤ 2026-05-31.\"}\n", "obj": {"answer": 42, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Counted shipments with customer_confirm_date present and ≤ 2026-05-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 34.39, "prompt_chars": 14569, "raw": "{\"answer\": 47, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Shipments with customer_confirm_date on/before 2026-06-30.\"}\n", "obj": {"answer": 47, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date on/before 2026-06-30."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 17.09, "prompt_chars": 14623, "raw": "{\"answer\": 82590, \"definition_id\": \"meridian.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum of gross-discount for 10 non-cancelled orders dated 2026-05-01..07-31.\"}\n", "obj": {"answer": 82590, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum of gross-discount for 10 non-cancelled orders dated 2026-05-01..07-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 25.04, "prompt_chars": 14572, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"365-day window (v1, effective through 2026-03-31); orders excl. cancelled, 2025-03-01–2026-02-28.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window (v1, effective through 2026-03-31); orders excl. cancelled, 2025-03-01–2026-02-28."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 25.73, "prompt_chars": 14572, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Orders (non-cancelled) in (2025-01-31,2026-01-31]; 21 distinct customers qualify.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Orders (non-cancelled) in (2025-01-31,2026-01-31]; 21 distinct customers qualify."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "prose", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 265.08, "prompt_chars": 12483, "raw": "{\"answer\": 23, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered=customer_confirm_date; on-time vs revised (or original) promised date.\"}\n", "obj": {"answer": 23, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered=customer_confirm_date; on-time vs revised (or original) promised date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 22.58, "prompt_chars": 14572, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"365-day window (2025-02-16 to 2026-02-15), non-cancelled orders, 21 customers.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window (2025-02-16 to 2026-02-15), non-cancelled orders, 21 customers."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 46.98, "prompt_chars": 14572, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"365-day window; excludes C013, C019, C022 (no qualifying orders).\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window; excludes C013, C019, C022 (no qualifying orders)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 29.03, "prompt_chars": 14572, "raw": "{\"answer\": 22, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Customers with non-cancelled order in (2025-03-31, 2026-03-31], 365-day window.\"}\n", "obj": {"answer": 22, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Customers with non-cancelled order in (2025-03-31, 2026-03-31], 365-day window."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 37.93, "prompt_chars": 14577, "raw": "{\"answer\": 10, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Used meridian delivered v1 (confirm_date≤D) and on_time v1 (vs promised_date only).\"}\n", "obj": {"answer": 10, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Used meridian delivered v1 (confirm_date≤D) and on_time v1 (vs promised_date only)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 98.54, "prompt_chars": 14577, "raw": "{\"answer\": 32, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Delivered per supplier: HAL=carrier_scan, ZEN=confirm; on-time uses revised/promised date.\"}\n", "obj": {"answer": 32, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Delivered per supplier: HAL=carrier_scan, ZEN=confirm; on-time uses revised/promised date."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 43.42, "prompt_chars": 14577, "raw": "{\"answer\": 9, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"27 delivered (confirm_date≤12/31); 9 met promised_date per v1 rule.\"}\n", "obj": {"answer": 9, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "27 delivered (confirm_date≤12/31); 9 met promised_date per v1 rule."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 102.07, "prompt_chars": 14577, "raw": "I'll rely on my manual computation since tool access isn't available.\n\nBased on my calculation: 50 shipments are delivered as of 2026-07-31 (customer_confirm_date present and ≤ 2026-07-31), of which 22 meet the on-time criterion using the definition in force on that date (compare to revised_promised_date if present, else promised_date).\n\n{\"answer\": 22, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Delivered=confirm_date≤2026-07-31 (meridian v1); on-time v2 applies (post 2026-02-01).\"}\n", "obj": {"answer": 22, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Delivered=confirm_date≤2026-07-31 (meridian v1); on-time v2 applies (post 2026-02-01)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 37.94, "prompt_chars": 14572, "raw": "{\"answer\": 17, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Window 180d (2025-10-03,2026-04-01], orders excl. cancelled, 17 unique customers.\"}\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window 180d (2025-10-03,2026-04-01], orders excl. cancelled, 17 unique customers."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 20.24, "prompt_chars": 14635, "raw": "{\"answer\": 31, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Counted HAL shipments with carrier_scan_date on/before 2026-07-31.\"}\n", "obj": {"answer": 31, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "Counted HAL shipments with carrier_scan_date on/before 2026-07-31."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 58.1, "prompt_chars": 14577, "raw": "{\"answer\": 14, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Delivered per supplier scope (HAL=scan date, ZEN=confirm date); v1 on_time uses promised_date only.\"}\n", "obj": {"answer": 14, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Delivered per supplier scope (HAL=scan date, ZEN=confirm date); v1 on_time uses promised_date only."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 48.21, "prompt_chars": 14577, "raw": "{\"answer\": 13, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Delivered (meridian v1) shipments on time using revised/promised date (on_time v2).\"}\n", "obj": {"answer": 13, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Delivered (meridian v1) shipments on time using revised/promised date (on_time v2)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 43.59, "prompt_chars": 14638, "raw": "{\"answer\": 33, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"33 C-118 shipments have carrier_scan_date on/before 2026-08-31.\"}\n", "obj": {"answer": 33, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "33 C-118 shipments have carrier_scan_date on/before 2026-08-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 39.72, "prompt_chars": 14627, "raw": "{\"answer\": 13, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Used halden.def.delivered v2 (carrier_scan_date) with on_time v2 for delivery status.\"}\n", "obj": {"answer": 13, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Used halden.def.delivered v2 (carrier_scan_date) with on_time v2 for delivery status."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 29.79, "prompt_chars": 14674, "raw": "{\"answer\": 34590, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum of gross-discount+freight for 5 non-cancelled HAL orders in range.\"}\n", "obj": {"answer": 34590, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum of gross-discount+freight for 5 non-cancelled HAL orders in range."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 71.59, "prompt_chars": 14638, "raw": "{\"answer\": 31, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Counted C-118 order shipments with carrier_scan_date on/before 2026-06-30.\"}\n", "obj": {"answer": 31, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "Counted C-118 order shipments with carrier_scan_date on/before 2026-06-30."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 36.28, "prompt_chars": 14674, "raw": "{\"answer\": 73715, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Summed gross-discount+freight for non-cancelled HAL orders in Q1 2026.\"}\n", "obj": {"answer": 73715, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Summed gross-discount+freight for non-cancelled HAL orders in Q1 2026."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 24.02, "prompt_chars": 14689, "raw": "{\"answer\": 9, \"definition_id\": \"halden.def.business_day\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Counted weekdays 2026-03-27 to 2026-04-11, excluding Halden holidays 04-03,04-06.\"}\n", "obj": {"answer": 9, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Counted weekdays 2026-03-27 to 2026-04-11, excluding Halden holidays 04-03,04-06."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 18.37, "prompt_chars": 14689, "raw": "{\"answer\": 11, \"definition_id\": \"halden.def.business_day\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"11 weekdays (no Halden holidays) between 2026-03-12 and 2026-03-27.\"}\n", "obj": {"answer": 11, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "11 weekdays (no Halden holidays) between 2026-03-12 and 2026-03-27."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 7.19, "prompt_chars": 14579, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No churn definition exists in the catalogue to compute this.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No churn definition exists in the catalogue to compute this."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 6.62, "prompt_chars": 14558, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No gross margin/cost-of-goods definition or data provided.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No gross margin/cost-of-goods definition or data provided."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 5.97, "prompt_chars": 14573, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No damage-in-transit field or definition exists in the data.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damage-in-transit field or definition exists in the data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 62.51, "prompt_chars": 14627, "raw": "{\"answer\": 12, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Used carrier_scan_date delivery + on_time v2 (revised/promised date) for HAL/C-118.\"}\n", "obj": {"answer": 12, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "Used carrier_scan_date delivery + on_time v2 (revised/promised date) for HAL/C-118."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 4.9, "prompt_chars": 14585, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No customer satisfaction score data provided in tables.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score data provided in tables."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 26.65, "prompt_chars": 14647, "raw": "{\"answer\": 43800, \"definition_id\": \"meridian.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum of gross-discount for 5 ZEN non-cancelled orders, Jun-Aug 2026.\"}\n", "obj": {"answer": 43800, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum of gross-discount for 5 ZEN non-cancelled orders, Jun-Aug 2026."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 77.64, "prompt_chars": 14627, "raw": "{\"answer\": 12, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Used halden.def.delivered v2 (carrier_scan_date) plus on_time v2; 16 delivered, 12 on time.\"}\n", "obj": {"answer": 12, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Used halden.def.delivered v2 (carrier_scan_date) plus on_time v2; 16 delivered, 12 on time."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 11.93, "prompt_chars": 14575, "raw": "{\"answer\": null, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": true, \"note\": \"Requires future order data beyond 2026-09-06; cannot be known yet.\"}\n", "obj": {"answer": null, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": true, "note": "Requires future order data beyond 2026-09-06; cannot be known yet."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 28.56, "prompt_chars": 16169, "raw": "{\"answer\": 14, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"180-day window (v2, effective 2026-04-01); orders 2026-03-05 to 2026-08-31, non-cancelled.\"}\n", "obj": {"answer": 14, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day window (v2, effective 2026-04-01); orders 2026-03-05 to 2026-08-31, non-cancelled."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 21.64, "prompt_chars": 16186, "raw": "{\"answer\": 109940, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum invoice (gross-discount) Apr-Jun 2026 minus returns in period.\"}\n", "obj": {"answer": 109940, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum invoice (gross-discount) Apr-Jun 2026 minus returns in period."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 30.45, "prompt_chars": 16169, "raw": "{\"answer\": 15, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Orders (non-cancelled) in (2025-12-03,2026-06-01], 15 distinct customers.\"}\n", "obj": {"answer": 15, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Orders (non-cancelled) in (2025-12-03,2026-06-01], 15 distinct customers."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 65.94, "prompt_chars": 14601, "raw": "{\"answer\": 28, \"definition_id\": \"meridian.def.delivered (ZEN shipments) / halden.def.delivered (HAL shipments)\", \"definition_version\": null, \"unanswerable\": false, \"note\": \"ZEN uses confirm_date rule v1; HAL uses scan_date rule v2, both ≤2026-08-31.\"}\n", "obj": {"answer": 28, "definition_id": "meridian.def.delivered (ZEN shipments) / halden.def.delivered (HAL shipments)", "definition_version": null, "unanswerable": false, "note": "ZEN uses confirm_date rule v1; HAL uses scan_date rule v2, both ≤2026-08-31."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 20.05, "prompt_chars": 16186, "raw": "{\"answer\": 37422, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum(gross-discount) of invoices in period minus returns in period.\"}\n", "obj": {"answer": 37422, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum(gross-discount) of invoices in period minus returns in period."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 22.43, "prompt_chars": 16186, "raw": "{\"answer\": 33020, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum(gross-discount) for July-Aug invoices minus Aug return R007 (11300).\"}\n", "obj": {"answer": 33020, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum(gross-discount) for July-Aug invoices minus Aug return R007 (11300)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 49.55, "prompt_chars": 16169, "raw": "{\"answer\": 16, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"180-day window (v2, effective 2026-04-01) applied to non-cancelled orders.\"}\n", "obj": {"answer": 16, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day window (v2, effective 2026-04-01) applied to non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 39.12, "prompt_chars": 16166, "raw": "{\"answer\": 58, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"ZEN uses meridian v1 (confirm date); HAL uses halden v2 (fed.C-118) via carrier scan.\"}\n", "obj": {"answer": 58, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "ZEN uses meridian v1 (confirm date); HAL uses halden v2 (fed.C-118) via carrier scan."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 45.76, "prompt_chars": 16166, "raw": "{\"answer\": 47, \"definition_id\": \"meridian.def.delivered / halden.def.delivered (fed.C-118.delivered)\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"HAL/C-118 uses carrier_scan_date (halden v2); ZEN uses customer_confirm_date (meridian v1).\"}\n", "obj": {"answer": 47, "definition_id": "meridian.def.delivered / halden.def.delivered (fed.C-118.delivered)", "definition_version": 1, "unanswerable": false, "note": "HAL/C-118 uses carrier_scan_date (halden v2); ZEN uses customer_confirm_date (meridian v1)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 27.93, "prompt_chars": 16220, "raw": "{\"answer\": 82590, \"definition_id\": \"meridian.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum gross-discount for 10 non-cancelled orders May-Jul 2026; general metric, meridian default.\"}\n", "obj": {"answer": 82590, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum gross-discount for 10 non-cancelled orders May-Jul 2026; general metric, meridian default."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 70.14, "prompt_chars": 16166, "raw": "{\"answer\": 55, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Hybrid: halden.def.delivered v2 (carrier_scan) for HAL/C-118 shipments via federation, meridian v1 (confirm date) otherwise.\"}\n", "obj": {"answer": 55, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Hybrid: halden.def.delivered v2 (carrier_scan) for HAL/C-118 shipments via federation, meridian v1 (confirm date) otherwise."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 121.4, "prompt_chars": 14627, "raw": "{\"answer\": 18, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Used delivered/on_time v2 + meridian business_day calendar; >2 business days late.\"}\n", "obj": {"answer": 18, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Used delivered/on_time v2 + meridian business_day calendar; >2 business days late."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 23.05, "prompt_chars": 16169, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"365-day window (2025-02-28,2026-02-28], excluding cancelled orders.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window (2025-02-28,2026-02-28], excluding cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 19.47, "prompt_chars": 16169, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"365-day window (2025-02-01–2026-01-31), non-cancelled orders, distinct customers.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window (2025-02-01–2026-01-31), non-cancelled orders, distinct customers."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 22.33, "prompt_chars": 16169, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"365-day window (2025-03-16 to 2026-03-15), non-cancelled orders, 21 customers qualify.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window (2025-03-16 to 2026-03-15), non-cancelled orders, 21 customers qualify."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 43.76, "prompt_chars": 16169, "raw": "{\"answer\": 22, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"365-day window (v1, effective through 2026-03-31); only C013,C019 inactive.\"}\n", "obj": {"answer": 22, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window (v1, effective through 2026-03-31); only C013,C019 inactive."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 45.52, "prompt_chars": 16169, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Distinct customers with non-cancelled orders in prior 365 days.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Distinct customers with non-cancelled orders in prior 365 days."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 125.84, "prompt_chars": 16174, "raw": "{\"answer\": 32, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Delivered per fed-pinned halden.def.delivered v2 (HAL/C-118) or meridian v1 (ZEN/C-204); on-time via meridian v2.\"}\n", "obj": {"answer": 32, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Delivered per fed-pinned halden.def.delivered v2 (HAL/C-118) or meridian v1 (ZEN/C-204); on-time via meridian v2."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 130.04, "prompt_chars": 16174, "raw": "{\"answer\": 34, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Combined meridian v2 on-time with Halden delivered/on-time for C-118/HAL per federation rules.\"}\n", "obj": {"answer": 34, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Combined meridian v2 on-time with Halden delivered/on-time for C-118/HAL per federation rules."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 41.96, "prompt_chars": 16169, "raw": "{\"answer\": 17, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"180-day window (2026-04-01 version), orders (2025-10-03,2026-04-01], excl. cancelled.\"}\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day window (2026-04-01 version), orders (2025-10-03,2026-04-01], excl. cancelled."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 34.03, "prompt_chars": 16235, "raw": "{\"answer\": 33, \"definition_id\": \"fed.C-118.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Halden delivered def (carrier_scan_date ≤ D) governs C-118 per federation record.\"}\n", "obj": {"answer": 33, "definition_id": "fed.C-118.delivered", "definition_version": 2, "unanswerable": false, "note": "Halden delivered def (carrier_scan_date ≤ D) governs C-118 per federation record."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 90.05, "prompt_chars": 16174, "raw": "{\"answer\": 15, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Delivered per fed.C-118/HAL rules; on-time v1 (pre-2026-02) vs promised_date.\"}\n", "obj": {"answer": 15, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Delivered per fed.C-118/HAL rules; on-time v1 (pre-2026-02) vs promised_date."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 119.85, "prompt_chars": 16174, "raw": "{\"answer\": 16, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Applied fed rules: HAL/C-118 use carrier_scan_date, others customer_confirm_date; promised_date only.\"}\n", "obj": {"answer": 16, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Applied fed rules: HAL/C-118 use carrier_scan_date, others customer_confirm_date; promised_date only."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 105.89, "prompt_chars": 16174, "raw": "{\"answer\": 15, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Delivered via halden/meridian scope rules; on-time uses v1 promised_date comparison.\"}\n", "obj": {"answer": 15, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Delivered via halden/meridian scope rules; on-time uses v1 promised_date comparison."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 51.45, "prompt_chars": 16232, "raw": "{\"answer\": 32, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Per fed.C-118.delivered, HAL shipments use carrier_scan_date ≤2026-07-31.\"}\n", "obj": {"answer": 32, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "Per fed.C-118.delivered, HAL shipments use carrier_scan_date ≤2026-07-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 73.53, "prompt_chars": 16235, "raw": "{\"answer\": 31, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"33 C-118 shipments exist; 2 scanned after cutoff, so 31 delivered.\"}\n", "obj": {"answer": 31, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "33 C-118 shipments exist; 2 scanned after cutoff, so 31 delivered."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 104.91, "prompt_chars": 16174, "raw": "{\"answer\": 22, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Fed rules (halden delivered) for C-118/HAL; meridian for C-204+ZEN; v2 pinned as current.\"}\n", "obj": {"answer": 22, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Fed rules (halden delivered) for C-118/HAL; meridian for C-204+ZEN; v2 pinned as current."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 32.31, "prompt_chars": 16271, "raw": "{\"answer\": 34590, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum gross-discount+freight for 5 HAL orders, May-Jul 2026, non-cancelled.\"}\n", "obj": {"answer": 34590, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum gross-discount+freight for 5 HAL orders, May-Jul 2026, non-cancelled."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 34.07, "prompt_chars": 16271, "raw": "{\"answer\": 73715, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum gross-discount+freight for HAL orders Jan1-Mar31 2026, non-cancelled.\"}\n", "obj": {"answer": 73715, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum gross-discount+freight for HAL orders Jan1-Mar31 2026, non-cancelled."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 26.61, "prompt_chars": 16286, "raw": "{\"answer\": 9, \"definition_id\": \"halden.def.business_day\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Business days 2026-03-27 to 2026-04-11, excl. Sat/Sun & Halden holidays.\"}\n", "obj": {"answer": 9, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Business days 2026-03-27 to 2026-04-11, excl. Sat/Sun & Halden holidays."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 72.51, "prompt_chars": 16224, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Per fed.C-118 rules: HAL carrier_scan delivery, on-time vs revised/promised date.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Per fed.C-118 rules: HAL carrier_scan delivery, on-time vs revised/promised date."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 93.89, "prompt_chars": 16224, "raw": "{\"answer\": 22, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Per fed.C-118 pins: HAL delivered=carrier_scan_date; on_time uses revised/promised date.\"}\n", "obj": {"answer": 22, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Per fed.C-118 pins: HAL delivered=carrier_scan_date; on_time uses revised/promised date."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 21.41, "prompt_chars": 16286, "raw": "{\"answer\": 11, \"definition_id\": \"halden.def.business_day\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Business days Mar13–Mar27 2026, no holidays fall in range.\"}\n", "obj": {"answer": 11, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Business days Mar13–Mar27 2026, no holidays fall in range."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 7.2, "prompt_chars": 16176, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No churn definition exists; contract_status has no change date.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No churn definition exists; contract_status has no change date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 58.17, "prompt_chars": 16224, "raw": "{\"answer\": 21, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"31 C-118 shipments delivered (carrier scan) by 2026-06-30; 21 on time per fed rules.\"}\n", "obj": {"answer": 21, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "31 C-118 shipments delivered (carrier scan) by 2026-06-30; 21 on time per fed rules."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 7.71, "prompt_chars": 16155, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No gross margin definition or cost data exists in catalogue/tables.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No gross margin definition or cost data exists in catalogue/tables."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 5.78, "prompt_chars": 16170, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No data field records damage-in-transit status for shipments.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No data field records damage-in-transit status for shipments."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 5.55, "prompt_chars": 16182, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No customer satisfaction score data provided in tables.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score data provided in tables."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 25.47, "prompt_chars": 16198, "raw": "{\"answer\": 28, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"C-204 not federated; used meridian delivered def (confirm_date ≤ 2026-08-31).\"}\n", "obj": {"answer": 28, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "C-204 not federated; used meridian delivered def (confirm_date ≤ 2026-08-31)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 27.0, "prompt_chars": 16244, "raw": "{\"answer\": 43800, \"definition_id\": \"meridian.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"ZEN not covered by federation; used meridian order_value (gross - discount).\"}\n", "obj": {"answer": 43800, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "ZEN not covered by federation; used meridian order_value (gross - discount)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 24.73, "prompt_chars": 16172, "raw": "{\"answer\": 3, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Orders (non-cancelled) in (2026-07-05,2027-01-01]: C001,C018,C020.\"}\n", "obj": {"answer": 3, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Orders (non-cancelled) in (2026-07-05,2027-01-01]: C001,C018,C020."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-sonnet-5", "arm": "vercy_fed", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 75.06, "prompt_chars": 16224, "raw": "{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No definition exists for a customer-specific working/business-day calendar.\"}\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No definition exists for a customer-specific working/business-day calendar."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}