{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-03", "family": "XORG", "rep": 1, "latency_s": 33.03, "prompt_chars": 12589, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"delivery\",\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Counted C-118 orders with customer_confirm_date ≤ 2026-07-31\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "delivery", "definition_version": null, "unanswerable": false, "note": "Counted C-118 orders with customer_confirm_date ≤ 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 49.28, "prompt_chars": 15071, "raw": "```json\n{\n  \"answer\": 109343,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Version 1 applied (effective to 2026-04-30); excludes freight and tax.\"\n}\n```\n", "obj": {"answer": 109343, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Version 1 applied (effective to 2026-04-30); excludes freight and tax."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-02", "family": "COMP", "rep": 1, "latency_s": 53.41, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day window from 2026-01-17 to 2026-07-15 inclusive\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day window from 2026-01-17 to 2026-07-15 inclusive"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-08", "family": "XORG", "rep": 1, "latency_s": 57.54, "prompt_chars": 15180, "raw": "```json\n{\"answer\": 18220, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"O004 + O024; calculated as (gross - discount + freight)\"}\n```\n", "obj": {"answer": 18220, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "O004 + O024; calculated as (gross - discount + freight)"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-04", "family": "COMP", "rep": 1, "latency_s": 61.81, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 16,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Window: Nov 17, 2025 – May 15, 2026. 16 customers with non-cancelled orders.\"\n}\n```\n", "obj": {"answer": 16, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window: Nov 17, 2025 – May 15, 2026. 16 customers with non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-05", "family": "XORG", "rep": 1, "latency_s": 61.92, "prompt_chars": 12597, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": \"on-time-delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-118 HAL shipments confirmed delivered by 2026-07-31; 11 total, 6 met promised/revised dates\"\n}\n```\n", "obj": {"answer": 6, "definition_id": "on-time-delivery", "definition_version": 1, "unanswerable": false, "note": "C-118 HAL shipments confirmed delivered by 2026-07-31; 11 total, 6 met promised/revised dates"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 79.72, "prompt_chars": 16796, "raw": "```json\n{\"answer\": 23, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"23 delivered shipments were on-time as of 2026-08-31\"}\n```\n", "obj": {"answer": 23, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "23 delivered shipments were on-time as of 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 28.13, "prompt_chars": 11329, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Counted weekdays (Mon-Fri) from 2025-12-29 through 2026-01-06 inclusive\"\n}\n```\n", "obj": {"answer": 7, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted weekdays (Mon-Fri) from 2025-12-29 through 2026-01-06 inclusive"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-04", "family": "ABST", "rep": 1, "latency_s": 7.77, "prompt_chars": 11131, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer satisfaction score field in provided data.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score field in provided data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 97.87, "prompt_chars": 15107, "raw": "```json\n{\n  \"answer\": 8,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"8 of 12 delivered C-118 shipments were on time\"\n}\n```\n", "obj": {"answer": 8, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "8 of 12 delivered C-118 shipments were on time"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 39.09, "prompt_chars": 15059, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"7 of 20 delivered shipments met promised_date requirement\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "7 of 20 delivered shipments met promised_date requirement"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 70.88, "prompt_chars": 12595, "raw": "```json\n{\n  \"answer\": 153700,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of gross_amount for non-cancelled orders, 2026-05-01 to 2026-07-31.\"\n}\n```\n", "obj": {"answer": 153700, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of gross_amount for non-cancelled orders, 2026-05-01 to 2026-07-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 94.41, "prompt_chars": 16844, "raw": "```json\n{\n  \"answer\": 8,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"8 of 13 C-118 HAL deliveries as of 2026-06-30 on time\"\n}\n```\n", "obj": {"answer": 8, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "8 of 13 C-118 HAL deliveries as of 2026-06-30 on time"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-11", "family": "COMP", "rep": 1, "latency_s": 47.99, "prompt_chars": 15059, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"20 of 44 delivered shipments were on time as of 2026-07-31\"\n}\n```\n", "obj": {"answer": 20, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "20 of 44 delivered shipments were on time as of 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-07", "family": "COMP", "rep": 1, "latency_s": 41.08, "prompt_chars": 16458, "raw": "```json\n{\n  \"answer\": 50,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Counted shipments with customer_confirm_date on/before 2026-08-31\"\n}\n```\n", "obj": {"answer": 50, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Counted shipments with customer_confirm_date on/before 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 60.42, "prompt_chars": 14033, "raw": "```json\n{\"answer\": 139070, \"definition_id\": \"Order value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"15 non-cancelled orders placed between 2026-05-01 and 2026-07-31\"}\n```\n", "obj": {"answer": 139070, "definition_id": "Order value", "definition_version": 1, "unanswerable": false, "note": "15 non-cancelled orders placed between 2026-05-01 and 2026-07-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-02", "family": "TEMP", "rep": 1, "latency_s": 16.79, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active = contract_status='open'. No historical contract status dates available.\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = contract_status='open'. No historical contract status dates available."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-04", "family": "TEMP", "rep": 1, "latency_s": 11.63, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with contract_status='open' as of 2026-02-15\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with contract_status='open' as of 2026-02-15"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 94.44, "prompt_chars": 12670, "raw": "```json\n{\n  \"answer\": 34406,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of invoiced amounts for four HAL-supplied C-118 orders placed 2026-05-01 to 2026-07-31\"\n}\n```\n", "obj": {"answer": 34406, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of invoiced amounts for four HAL-supplied C-118 orders placed 2026-05-01 to 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 60.72, "prompt_chars": 14038, "raw": "```json\n{\"answer\": 10, \"definition_id\": \"Delivered_Shipments\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Meridian rule: customer_confirm_date present and on/before 2026-08-31\"}\n```\n", "obj": {"answer": 10, "definition_id": "Delivered_Shipments", "definition_version": 1, "unanswerable": false, "note": "Meridian rule: customer_confirm_date present and on/before 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 19.43, "prompt_chars": 16866, "raw": "```json\n{\n  \"answer\": 60055,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Seven ZEN orders placed 2026-06-01 to 2026-08-31; all non-cancelled.\"\n}\n```\n", "obj": {"answer": 60055, "definition_id": "meridian.def.order_value", "definition_version": 2, "unanswerable": false, "note": "Seven ZEN orders placed 2026-06-01 to 2026-08-31; all non-cancelled."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 50.48, "prompt_chars": 15071, "raw": "```json\n{\"answer\": 104670, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"12 invoices in period; 1 return (R006) dated 2026-06-04.\"}\n```\n", "obj": {"answer": 104670, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "12 invoices in period; 1 return (R006) dated 2026-06-04."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 34.58, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"18 customers with non-cancelled orders in 180-day window (2026-03-04 to 2026-08-31]\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "18 customers with non-cancelled orders in 180-day window (2026-03-04 to 2026-08-31]"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-04", "family": "XORG", "rep": 1, "latency_s": 138.04, "prompt_chars": 16514, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"10 of 17 C-118 HAL shipments delivered on time by 2026-08-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "10 of 17 C-118 HAL shipments delivered on time by 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 56.92, "prompt_chars": 16639, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"9 business days: 2025-12-31, 2026-01-02, 05, 07-09, 12-14\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "9 business days: 2025-12-31, 2026-01-02, 05, 07-09, 12-14"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-07", "family": "TEMP", "rep": 1, "latency_s": 49.13, "prompt_chars": 15071, "raw": "```json\n{\n  \"answer\": 109343,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Invoiced orders sum: 112920; returns deducted: 3577\"\n}\n```\n", "obj": {"answer": 109343, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Invoiced orders sum: 112920; returns deducted: 3577"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-05", "family": "XORG", "rep": 1, "latency_s": 26.29, "prompt_chars": 16844, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"9 of 15 C-118 shipments delivered by 2026-07-31 were on time\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "9 of 15 C-118 shipments delivered by 2026-07-31 were on time"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 42.77, "prompt_chars": 15071, "raw": "```json\n{\n  \"answer\": 115288,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Sum of (gross - discount) for 13 invoices dated 2025-10-01 to 2025-12-31, minus 1 return dated in period\"\n}\n```\n", "obj": {"answer": 115288, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum of (gross - discount) for 13 invoices dated 2025-10-01 to 2025-12-31, minus 1 return dated in period"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 8.49, "prompt_chars": 12524, "raw": "```json\n{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No customer satisfaction score field in provided data.\"}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score field in provided data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-02", "family": "XORG", "rep": 1, "latency_s": 53.02, "prompt_chars": 16836, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Fed.C-118.delivered governs; counted HAL shipments with carrier_scan_date≤2026-06-30\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "Fed.C-118.delivered governs; counted HAL shipments with carrier_scan_date≤2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 56.48, "prompt_chars": 16761, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Orders dated 2026-07-06 to 2027-01-01: C001, C004, C011, C013, C022\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Orders dated 2026-07-06 to 2027-01-01: C001, C004, C011, C013, C022"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 43.74, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"active_customers\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day window from Dec 4, 2025 to Jun 1, 2026; non-cancelled orders only.\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "active_customers", "definition_version": 2, "unanswerable": false, "note": "180-day window from Dec 4, 2025 to Jun 1, 2026; non-cancelled orders only."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 27.71, "prompt_chars": 15129, "raw": "```json\n{\n  \"answer\": 58155,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"7 non-cancelled ZEN orders 2026-06-01 to 2026-08-31\"\n}\n```\n", "obj": {"answer": 58155, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "7 non-cancelled ZEN orders 2026-06-01 to 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 38.95, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Window: (2025-12-03, 2026-06-01]. Customers: C001, C003, C004, C005, C007, C008, C009, C010, C011, C012, C013, C014, C015, C017, C020, C023, C024.\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window: (2025-12-03, 2026-06-01]. Customers: C001, C003, C004, C005, C007, C008, C009, C010, C011, C012, C013, C014, C015, C017, C020, C023, C024."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 26.93, "prompt_chars": 16836, "raw": "```json\n{\"answer\": 14, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"C-118 shipments with carrier_scan_date on or before 2026-08-31\"}\n```\n", "obj": {"answer": 14, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "C-118 shipments with carrier_scan_date on or before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-12", "family": "COMP", "rep": 1, "latency_s": 24.57, "prompt_chars": 15105, "raw": "```json\n{\n  \"answer\": 126340,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"14 non-cancelled orders placed in period; order value = gross - discount\"\n}\n```\n", "obj": {"answer": 126340, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "14 non-cancelled orders placed in period; order value = gross - discount"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-06", "family": "COMP", "rep": 1, "latency_s": 43.73, "prompt_chars": 13999, "raw": "```json\n{\n  \"answer\": 55463,\n  \"definition_id\": \"net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Freight included; period after 1 May 2026. Five invoices, one return.\"\n}\n```\n", "obj": {"answer": 55463, "definition_id": "net_revenue", "definition_version": 2, "unanswerable": false, "note": "Freight included; period after 1 May 2026. Five invoices, one return."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-07", "family": "XORG", "rep": 1, "latency_s": 57.16, "prompt_chars": 15180, "raw": "```json\n{\n  \"answer\": 29065,\n  \"definition_id\": \"halden.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"O009,O045,O065,O072: (4600-460+350)+(3100-310+150)+(16100-2415+200)+(7500-0+250)\"\n}\n```\n", "obj": {"answer": 29065, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "O009,O045,O065,O072: (4600-460+350)+(3100-310+150)+(16100-2415+200)+(7500-0+250)"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-03", "family": "TEMP", "rep": 1, "latency_s": 57.51, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Active in 365-day window ending 2026-03-15, non-cancelled orders\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Active in 365-day window ending 2026-03-15, non-cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 42.93, "prompt_chars": 14108, "raw": "```json\n{\n  \"answer\": 18220,\n  \"definition_id\": \"Order value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden formula: gross minus discount plus freight, tax excluded\"\n}\n```\n\n**Calculation:**\n\nQualifying orders (HAL supplier, C-118 contract, placed 2026-01-01 to 2026-03-31, non-cancelled):\n\n1. **O004** (2026-01-23): 7500 − 375 + 150 = **7,275**\n2. **O024** (2026-02-26): 12700 − 1905 + 150 = **10,945**\n\n**Total: 18,220**\n\nNote: O051 (2026-03-19) is excluded due to cancelled status.\n", "obj": {"answer": 18220, "definition_id": "Order value", "definition_version": 1, "unanswerable": false, "note": "Halden formula: gross minus discount plus freight, tax excluded"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-12", "family": "COMP", "rep": 1, "latency_s": 23.26, "prompt_chars": 14033, "raw": "```json\n{\n  \"answer\": 139070,\n  \"definition_id\": \"Order value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"15 non-cancelled orders, 2026-05-01 to 2026-07-31: gross minus discount.\"\n}\n```\n", "obj": {"answer": 139070, "definition_id": "Order value", "definition_version": 1, "unanswerable": false, "note": "15 non-cancelled orders, 2026-05-01 to 2026-07-31: gross minus discount."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-08", "family": "XORG", "rep": 1, "latency_s": 22.91, "prompt_chars": 16917, "raw": "```json\n{\"answer\": 18220, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Fed record C-118: gross - discount + freight. Orders O004, O024.\"}\n```\n", "obj": {"answer": 18220, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Fed record C-118: gross - discount + freight. Orders O004, O024."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 57.82, "prompt_chars": 11173, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Business day calculation requires weekend/holiday data unavailable.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Business day calculation requires weekend/holiday data unavailable."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-12", "family": "COMP", "rep": 1, "latency_s": 35.85, "prompt_chars": 16842, "raw": "```json\n{\n  \"answer\": 139070,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"15 non-cancelled orders: gross minus discount\"\n}\n```\n", "obj": {"answer": 139070, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "15 non-cancelled orders: gross minus discount"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-03", "family": "TEMP", "rep": 1, "latency_s": 53.05, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"365-day window: 2025-03-15 to 2026-03-15. Non-cancelled orders only.\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "Active customers", "definition_version": 1, "unanswerable": false, "note": "365-day window: 2025-03-15 to 2026-03-15. Non-cancelled orders only."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-02", "family": "COMP", "rep": 1, "latency_s": 51.35, "prompt_chars": 16791, "raw": "```json\n{\"answer\": 17, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Counted unique customers with non-cancelled orders from 2026-01-17 to 2026-07-15\"}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Counted unique customers with non-cancelled orders from 2026-01-17 to 2026-07-15"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 43.25, "prompt_chars": 16808, "raw": "```json\n{\n  \"answer\": 104670,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Sum of (gross - discount + freight) for invoices 2026-05-01 to 2026-06-30, minus returns in period.\"\n}\n```\n", "obj": {"answer": 104670, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Sum of (gross - discount + freight) for invoices 2026-05-01 to 2026-06-30, minus returns in period."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-09", "family": "TEMP", "rep": 1, "latency_s": 15.65, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active = signup_date ≤ 2026-04-01 AND contract_status = open\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = signup_date ≤ 2026-04-01 AND contract_status = open"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-04", "family": "TEMP", "rep": 1, "latency_s": 40.72, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"20 customers with orders in (2025-02-15, 2026-02-15], excluding cancelled\"\n}\n```\n", "obj": {"answer": 20, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "20 customers with orders in (2025-02-15, 2026-02-15], excluding cancelled"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-06", "family": "XORG", "rep": 1, "latency_s": 39.34, "prompt_chars": 12597, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": \"revised_promised_date_method\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"On-time uses revised_promised_date when present, otherwise promised_date. Delivered = customer_confirm_date not null before 2026-06-30.\"\n}\n```\n", "obj": {"answer": 6, "definition_id": "revised_promised_date_method", "definition_version": 1, "unanswerable": false, "note": "On-time uses revised_promised_date when present, otherwise promised_date. Delivered = customer_confirm_date not null before 2026-06-30."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-06", "family": "TEMP", "rep": 1, "latency_s": 35.72, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"On-time delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Meridian rule pre-Feb 2026: delivery <= promised_date\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "On-time delivery", "definition_version": 1, "unanswerable": false, "note": "Meridian rule pre-Feb 2026: delivery <= promised_date"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 69.71, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"45 shipments delivered; 21 met on-time criteria using revised_promised_date\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "45 shipments delivered; 21 met on-time criteria using revised_promised_date"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 28.95, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Window: 2025-03-15 to 2026-03-15. 21 customers with non-cancelled orders.\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Window: 2025-03-15 to 2026-03-15. 21 customers with non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-08", "family": "TEMP", "rep": 1, "latency_s": 68.7, "prompt_chars": 13999, "raw": "```json\n{\n  \"answer\": 115288,\n  \"definition_id\": \"net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"13 invoices in period, freight excluded (pre-May 2026), 1 return subtracted.\"\n}\n```\n", "obj": {"answer": 115288, "definition_id": "net_revenue", "definition_version": 1, "unanswerable": false, "note": "13 invoices in period, freight excluded (pre-May 2026), 1 return subtracted."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 87.8, "prompt_chars": 12597, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"on_time_performance\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"5 of 11 C-118 HAL shipments delivered by 2026-06-30 were on-time\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "on_time_performance", "definition_version": 1, "unanswerable": false, "note": "5 of 11 C-118 HAL shipments delivered by 2026-06-30 were on-time"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-05", "family": "TEMP", "rep": 1, "latency_s": 47.17, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"on-time-delivery-meridian\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Applied 1 Jan–31 Jan 2026 rule: delivered if confirmed ≤ date; on time if confirmed ≤ promised date.\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "on-time-delivery-meridian", "definition_version": 1, "unanswerable": false, "note": "Applied 1 Jan–31 Jan 2026 rule: delivered if confirmed ≤ date; on time if confirmed ≤ promised date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 43.41, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"15 customers with non-cancelled orders in (2025-01-31, 2026-01-31]\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "15 customers with non-cancelled orders in (2025-01-31, 2026-01-31]"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-09", "family": "COMP", "rep": 1, "latency_s": 43.13, "prompt_chars": 16458, "raw": "```json\n{\n  \"answer\": 37,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Counted shipments with customer_confirm_date present and on/before 2026-05-31\"\n}\n```\n", "obj": {"answer": 37, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Counted shipments with customer_confirm_date present and on/before 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 29.15, "prompt_chars": 11148, "raw": "```json\n{\n  \"answer\": 50,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date on/before 2026-08-31\"\n}\n```\n", "obj": {"answer": 50, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with customer_confirm_date on/before 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 124.76, "prompt_chars": 16796, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Using revised_promised_date when present, else promised_date.\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Using revised_promised_date when present, else promised_date."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-09", "family": "XORG", "rep": 1, "latency_s": 21.64, "prompt_chars": 12722, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"business_day_standard\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"2025-12-30 order date to 2026-01-14 scan date; 10 business days excluding weekends and 2026-01-01 holiday\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "business_day_standard", "definition_version": 1, "unanswerable": false, "note": "2025-12-30 order date to 2026-01-14 scan date; 10 business days excluding weekends and 2026-01-01 holiday"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-10", "family": "XORG", "rep": 1, "latency_s": 70.97, "prompt_chars": 11329, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Weekdays 2025-12-29 through 2026-01-06, excluding weekends only\"\n}\n```\n", "obj": {"answer": 7, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Weekdays 2025-12-29 through 2026-01-06, excluding weekends only"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 81.22, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"365-day window: 2025-02-28 to 2026-02-28, non-cancelled orders\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "Active customers", "definition_version": null, "unanswerable": false, "note": "365-day window: 2025-02-28 to 2026-02-28, non-cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 50.5, "prompt_chars": 16478, "raw": "```json\n{\n  \"answer\": 115288,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"13 invoices in period; 1 return on 2025-11-03\"\n}\n```\n", "obj": {"answer": 115288, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "13 invoices in period; 1 return on 2025-11-03"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-03", "family": "XORG", "rep": 1, "latency_s": 35.18, "prompt_chars": 11196, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"C-118 orders with HAL supplier having confirmed delivery by 2026-07-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "C-118 orders with HAL supplier having confirmed delivery by 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 88.12, "prompt_chars": 15180, "raw": "```json\n{\"answer\": 29065, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Four non-cancelled C-118 orders: O009 (4490) + O045 (2940) + O065 (13885) + O072 (7750).\"}\n```\n", "obj": {"answer": 29065, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Four non-cancelled C-118 orders: O009 (4490) + O045 (2940) + O065 (13885) + O072 (7750)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 42.79, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"21 customers with non-cancelled orders in (2025-03-15, 2026-03-15].\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "21 customers with non-cancelled orders in (2025-03-15, 2026-03-15]."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 42.95, "prompt_chars": 14027, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"delivered_shipments_halden\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Halden rule: carrier scan date present and ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "delivered_shipments_halden", "definition_version": 2, "unanswerable": false, "note": "Halden rule: carrier scan date present and ≤ 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-01", "family": "COMP", "rep": 1, "latency_s": 52.9, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day lookback from 2026-08-31: orders 2026-03-05 to 2026-08-31, non-cancelled\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day lookback from 2026-08-31: orders 2026-03-05 to 2026-08-31, non-cancelled"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 26.92, "prompt_chars": 11226, "raw": "```json\n{\n  \"answer\": 57500,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"ZEN supplier, non-cancelled orders, 2026-06-01 to 2026-08-31, gross amounts\"\n}\n```\n", "obj": {"answer": 57500, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "ZEN supplier, non-cancelled orders, 2026-06-01 to 2026-08-31, gross amounts"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-11", "family": "COMP", "rep": 1, "latency_s": 93.65, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 22,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"22 delivered shipments were on-time by 2026-07-31\"\n}\n```\n", "obj": {"answer": 22, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "22 delivered shipments were on-time by 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 26.65, "prompt_chars": 13979, "raw": "```json\n{\"answer\": 42, \"definition_id\": \"Delivered shipments\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Meridian: customer confirmation date present and on/before 2026-06-30\"}\n```\n", "obj": {"answer": 42, "definition_id": "Delivered shipments", "definition_version": 1, "unanswerable": false, "note": "Meridian: customer confirmation date present and on/before 2026-06-30"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-01", "family": "XORG", "rep": 1, "latency_s": 83.04, "prompt_chars": 15099, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"All 18 C-118 shipments have carrier_scan_date ≤ 2026-08-31.\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "All 18 C-118 shipments have carrier_scan_date ≤ 2026-08-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 47.56, "prompt_chars": 12541, "raw": "```json\n{\"answer\": 50, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date populated and ≤ 2026-08-31\"}\n```\n", "obj": {"answer": 50, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date populated and ≤ 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 84.13, "prompt_chars": 13982, "raw": "Looking at this question, I need to apply the \"Active customers\" definition from the context.\n\nAs of 2026-05-15, I must determine which rule applies:\n- From 1 April 2026 onwards, the window is 180 days\n- The date 2026-05-15 falls after 1 April 2026, so I use the 180-day window\n\n**Calculating the window:**\n180 days before 2026-05-15:\n- Back through May (15 days), April (30), March (31), February (28), January (31), December (31) = 166 days\n- Need 14 more days into November: 2025-11-17\n\nWindow: 2025-11-17 through 2026-05-15 (inclusive)\n\n**Finding active customers:**\nA customer is active if they ", "obj": {"answer": 16, "definition_id": "Active customers", "definition_version": 1, "unanswerable": false, "note": "180-day window: Nov 17 2025 - May 15 2026"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 114.96, "prompt_chars": 11204, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"C-118 HAL shipments scanned on/before promised date.\"\n}\n```\n", "obj": {"answer": 9, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "C-118 HAL shipments scanned on/before promised date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-01", "family": "XORG", "rep": 1, "latency_s": 32.03, "prompt_chars": 12589, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"delivery_definition\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Counted C-118 shipments with customer_confirm_date populated and on or before 2026-08-31\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "delivery_definition", "definition_version": 1, "unanswerable": false, "note": "Counted C-118 shipments with customer_confirm_date populated and on or before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-01", "family": "COMP", "rep": 1, "latency_s": 14.82, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with contract_status='open' and signup_date ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with contract_status='open' and signup_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-10", "family": "TEMP", "rep": 1, "latency_s": 73.29, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Applied v2 effective 2026-02-01; used revised_promised_date when present.\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Applied v2 effective 2026-02-01; used revised_promised_date when present."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 24.84, "prompt_chars": 16744, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Gross margin undefined; cost of goods sold data unavailable\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Gross margin undefined; cost of goods sold data unavailable"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-02", "family": "TEMP", "rep": 1, "latency_s": 34.46, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 19,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Version 1 effective until 2026-03-31; window 365 days; interval (2025-01-31, 2026-01-31]\"\n}\n```\n", "obj": {"answer": 19, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Version 1 effective until 2026-03-31; window 365 days; interval (2025-01-31, 2026-01-31]"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-06", "family": "XORG", "rep": 1, "latency_s": 104.35, "prompt_chars": 16514, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"9 delivered shipments; 5 on time per revised/promised dates\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "9 delivered shipments; 5 on time per revised/promised dates"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-01", "family": "TEMP", "rep": 1, "latency_s": 15.36, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active = contract_status 'open' with signup before 2026-02-28\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = contract_status 'open' with signup before 2026-02-28"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-03", "family": "COMP", "rep": 1, "latency_s": 59.19, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 22,\n  \"definition_id\": \"customers_ordered_past_year\",\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders between 2025-06-01 and 2026-06-01\"\n}\n```\n", "obj": {"answer": 22, "definition_id": "customers_ordered_past_year", "definition_version": null, "unanswerable": false, "note": "Customers with orders between 2025-06-01 and 2026-06-01"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-01", "family": "ABST", "rep": 1, "latency_s": 120.43, "prompt_chars": 15028, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Churn inferred as inactive transition\"\n}\n```\n", "obj": {"answer": 6, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Churn inferred as inactive transition"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-05", "family": "ABST", "rep": 1, "latency_s": 81.68, "prompt_chars": 15076, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Customer business day calendar not defined in catalogue\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Customer business day calendar not defined in catalogue"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-05", "family": "COMP", "rep": 1, "latency_s": 31.64, "prompt_chars": 11168, "raw": "```json\n{\n  \"answer\": 108915,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of invoices issued 2026-05-01 to 2026-06-30 minus returns on those orders.\"\n}\n```\n", "obj": {"answer": 108915, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of invoices issued 2026-05-01 to 2026-06-30 minus returns on those orders."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 45.29, "prompt_chars": 16478, "raw": "```json\n{\n  \"answer\": 109343,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Sum (gross - discount) of invoices minus returns for Q1 2026\"\n}\n```\n", "obj": {"answer": 109343, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum (gross - discount) of invoices minus returns for Q1 2026"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 67.69, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Version 2 active (180-day window). Orders in (2026-01-16, 2026-07-15].\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Version 2 active (180-day window). Orders in (2026-01-16, 2026-07-15]."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-03", "family": "XORG", "rep": 1, "latency_s": 37.57, "prompt_chars": 16506, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"15 HAL-supplied C-118 shipments with carrier_scan_date on/before 2026-07-31\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "15 HAL-supplied C-118 shipments with carrier_scan_date on/before 2026-07-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-05", "family": "TEMP", "rep": 1, "latency_s": 44.34, "prompt_chars": 15059, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"10 delivered on-time shipments as of 2026-01-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "10 delivered on-time shipments as of 2026-01-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-07", "family": "COMP", "rep": 1, "latency_s": 29.35, "prompt_chars": 11148, "raw": "```json\n{\n  \"answer\": 50,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unaswerable\": false,\n  \"note\": \"Shipments with non-null customer_confirm_date on/before 2026-08-31\"\n}\n```\n", "obj": {"answer": 50, "definition_id": null, "definition_version": null, "unaswerable": false, "note": "Shipments with non-null customer_confirm_date on/before 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 73.62, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered = customer_confirm_date ≤ 2025-12-31; on-time = delivered date ≤ promised or revised date.\"\n}\n```\n", "obj": {"answer": 9, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date ≤ 2025-12-31; on-time = delivered date ≤ promised or revised date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-04", "family": "TEMP", "rep": 1, "latency_s": 76.35, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Window 365 days: 2025-02-15 to 2026-02-15, non-cancelled orders\"\n}\n```\n", "obj": {"answer": 20, "definition_id": "Active customers", "definition_version": 1, "unanswerable": false, "note": "Window 365 days: 2025-02-15 to 2026-02-15, non-cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 14.14, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active = open contract_status. Snapshot-based count.\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = open contract_status. Snapshot-based count."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 26.66, "prompt_chars": 11148, "raw": "```json\n{\n  \"answer\": 43,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date on/before 2026-06-30\"\n}\n```\n", "obj": {"answer": 43, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with customer_confirm_date on/before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-09", "family": "XORG", "rep": 1, "latency_s": 20.52, "prompt_chars": 16639, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Order O006 dated 2025-12-30, shipment S005 scanned 2026-01-14. Nine business days between (weekdays excluding 2026-01-01 and 2026-01-06 Halden holidays).\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Order O006 dated 2025-12-30, shipment S005 scanned 2026-01-14. Nine business days between (weekdays excluding 2026-01-01 and 2026-01-06 Halden holidays)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 134.26, "prompt_chars": 12561, "raw": "```json\n{\n  \"answer\": 90239,\n  \"definition_id\": \"meridian_net_revenue_order_period\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"9 invoiced orders placed 2026-05-01 to 2026-06-30; zero returns in period\"\n}\n```\n", "obj": {"answer": 90239, "definition_id": "meridian_net_revenue_order_period", "definition_version": 1, "unanswerable": false, "note": "9 invoiced orders placed 2026-05-01 to 2026-06-30; zero returns in period"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-10", "family": "XORG", "rep": 1, "latency_s": 31.07, "prompt_chars": 14160, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"Business days\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden calendar: 2025-12-29,30,31; 2026-01-02,05 are business days; 01-01,06 are holidays\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "Business days", "definition_version": 1, "unanswerable": false, "note": "Halden calendar: 2025-12-29,30,31; 2026-01-02,05 are business days; 01-01,06 are holidays"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 21.64, "prompt_chars": 15051, "raw": "```json\n{\"answer\": 43, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Shipments with customer_confirm_date on or before 2026-06-30\"}\n```\n", "obj": {"answer": 43, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date on or before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 28.5, "prompt_chars": 16788, "raw": "```json\n{\"answer\": 51, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Shipments with customer_confirm_date present and on/before 2026-08-31\"}\n```\n", "obj": {"answer": 51, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present and on/before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-11", "family": "XORG", "rep": 1, "latency_s": 31.69, "prompt_chars": 12600, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"delivery_definition\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Delivered = customer_confirm_date populated and <= 2026-08-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "delivery_definition", "definition_version": 1, "unanswerable": false, "note": "Delivered = customer_confirm_date populated and <= 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 39.73, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"on-time-delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"7 delivered shipments with customer confirmation on/before 2025-12-31 met promised dates\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "on-time-delivery", "definition_version": 1, "unanswerable": false, "note": "7 delivered shipments with customer confirmation on/before 2025-12-31 met promised dates"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-07", "family": "XORG", "rep": 1, "latency_s": 68.69, "prompt_chars": 11277, "raw": "```json\n{\n  \"answer\": 34406,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of invoice amounts for HAL C-118 orders 2026-05-01 to 2026-07-31\"\n}\n```\n\n**Calculation details:**\nFiltered orders meeting criteria (C-118, HAL supplier, placed 2026-05-01 to 2026-07-31 inclusive, non-cancelled):\n- O009 (2026-07-14): Invoice I007 = 5,276\n- O045 (2026-05-10): Invoice I035 = 3,470\n- O065 (2026-07-21): Invoice I051 = 16,485\n- O072 (2026-05-18): Invoice I057 = 9,175\n\n**Total: 34,406**\n", "obj": {"answer": 34406, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of invoice amounts for HAL C-118 orders 2026-05-01 to 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 22.11, "prompt_chars": 15007, "raw": "```json\n{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"Gross margin not defined; COGS data unavailable.\"}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Gross margin not defined; COGS data unavailable."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 173.91, "prompt_chars": 16514, "raw": "```json\n{\n  \"answer\": 11,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Used halden.def.delivered; carrier_scan_date as delivery date.\"\n}\n```\n", "obj": {"answer": 11, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Used halden.def.delivered; carrier_scan_date as delivery date."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 25.53, "prompt_chars": 16536, "raw": "```json\n{\n  \"answer\": 58155,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"7 non-cancelled ZEN orders placed 2026-06-01 to 2026-08-31\"\n}\n```\n", "obj": {"answer": 58155, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "7 non-cancelled ZEN orders placed 2026-06-01 to 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 50.12, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Version 1 applies: interval (2025-02-15, 2026-02-15], window_days=365\"\n}\n```\n", "obj": {"answer": 20, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Version 1 applies: interval (2025-02-15, 2026-02-15], window_days=365"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 23.54, "prompt_chars": 16917, "raw": "```json\n{\n  \"answer\": 21315,\n  \"definition_id\": \"halden.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-118 orders 2026-05-01 to 2026-07-31, non-cancelled, using Halden formula.\"\n}\n```\n", "obj": {"answer": 21315, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "C-118 orders 2026-05-01 to 2026-07-31, non-cancelled, using Halden formula."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-08", "family": "XORG", "rep": 1, "latency_s": 50.17, "prompt_chars": 16587, "raw": "```json\n{\"answer\": 18220, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"O004 (7275) and O024 (10945) within date range, non-cancelled\"}\n```\n", "obj": {"answer": 18220, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "O004 (7275) and O024 (10945) within date range, non-cancelled"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-08", "family": "TEMP", "rep": 1, "latency_s": 32.59, "prompt_chars": 11168, "raw": "```json\n{\n  \"answer\": 141511,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Invoiced amount (13 invoices) minus returns for Q4 2025\"\n}\n```\n", "obj": {"answer": 141511, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Invoiced amount (13 invoices) minus returns for Q4 2025"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 54.2, "prompt_chars": 12514, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Question asks future forecast beyond available data cutoff\"\n}\n```\n\nThe question requests a forecast as of 2027-01-01, which is future-dated beyond the available data (orders through ~2026-08-29). To answer \"how many active customers\" would exist by then using the stated definition of \"customers who ordered in the past year,\" I would need data through 2027-01-01. The available orders table does not contain that information, making prediction impossible.\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Question asks future forecast beyond available data cutoff"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-03", "family": "ABST", "rep": 1, "latency_s": 8.23, "prompt_chars": 11119, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No damage indicator field in shipments or returns data.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damage indicator field in shipments or returns data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-05", "family": "XORG", "rep": 1, "latency_s": 72.2, "prompt_chars": 16514, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"C-118 HAL orders: 18 with shipments. 15 delivered by 2026-07-31. 9 on-time.\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "C-118 HAL orders: 18 with shipments. 15 delivered by 2026-07-31. 9 on-time."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-01", "family": "XORG", "rep": 1, "latency_s": 41.12, "prompt_chars": 11196, "raw": "```json\n{\n  \"answer\": 14,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"C-118 HAL shipments with customer_confirm_date ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 14, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "C-118 HAL shipments with customer_confirm_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-11", "family": "COMP", "rep": 1, "latency_s": 99.12, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 19,\n  \"definition_id\": \"meridian-on-time-delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Applied Meridian delivered shipments and on-time rules.\"\n}\n```\n", "obj": {"answer": 19, "definition_id": "meridian-on-time-delivery", "definition_version": 1, "unanswerable": false, "note": "Applied Meridian delivered shipments and on-time rules."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-06", "family": "ABST", "rep": 1, "latency_s": 101.43, "prompt_chars": 13952, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": true,\n  \"note\": \"Question asks about future date; data current through 2026-09-06\"\n}\n```\n", "obj": {"answer": null, "definition_id": "Active customers", "definition_version": 1, "unanswerable": true, "note": "Question asks about future date; data current through 2026-09-06"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 106.27, "prompt_chars": 12589, "raw": "```json\n{\"answer\": 12, \"definition_id\": \"customer_confirm_date\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Halden C-118 orders with confirmed delivery by 2026-07-31\"}\n```\n", "obj": {"answer": 12, "definition_id": "customer_confirm_date", "definition_version": 1, "unanswerable": false, "note": "Halden C-118 orders with confirmed delivery by 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 39.64, "prompt_chars": 16587, "raw": "```json\n{\"answer\": 29065, \"definition_id\": \"halden.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Four qualifying orders: O009, O045, O065, O072\"}\n```\n", "obj": {"answer": 29065, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Four qualifying orders: O009, O045, O065, O072"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 38.42, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"past_year_orders\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders placed 2025-05-15 through 2026-05-15.\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "past_year_orders", "definition_version": 1, "unanswerable": false, "note": "Customers with orders placed 2025-05-15 through 2026-05-15."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-07", "family": "COMP", "rep": 1, "latency_s": 37.92, "prompt_chars": 13979, "raw": "```json\n{\n  \"answer\": 50,\n  \"definition_id\": \"delivered-shipments-meridian\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date present and on/before 2026-08-31\"\n}\n```\n", "obj": {"answer": 50, "definition_id": "delivered-shipments-meridian", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present and on/before 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-04", "family": "TEMP", "rep": 1, "latency_s": 48.22, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": \"customers_ordered_past_year\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders 2025-02-15 to 2026-02-15\"\n}\n```\n", "obj": {"answer": 20, "definition_id": "customers_ordered_past_year", "definition_version": 1, "unanswerable": false, "note": "Customers with orders 2025-02-15 to 2026-02-15"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-04", "family": "TEMP", "rep": 1, "latency_s": 49.5, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 14,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Interval (2025-02-15, 2026-02-15], 365-day window, non-cancelled orders\"\n}\n```\n", "obj": {"answer": 14, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Interval (2025-02-15, 2026-02-15], 365-day window, non-cancelled orders"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-09", "family": "XORG", "rep": 1, "latency_s": 81.38, "prompt_chars": 14160, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"Business days\",\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Halden's 2026 holidays applied; weekdays excluded 2026-01-01, 2026-01-06\"\n}\n```\n\nCalculation: Order O006 dated 2025-12-30, shipment S005 carrier scan 2026-01-14. Business days from 2025-12-31 to 2026-01-14 inclusive, using Halden's calendar (holidays: 1 Jan, 6 Jan): Dec 31, Jan 2, 5, 7, 8, 9, 12, 13, 14 = 9 weekdays excluding Halden holidays and weekends.\n", "obj": {"answer": 9, "definition_id": "Business days", "definition_version": null, "unanswerable": false, "note": "Halden's 2026 holidays applied; weekdays excluded 2026-01-01, 2026-01-06"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 45.11, "prompt_chars": 15099, "raw": "```json\n{\"answer\": 16, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"16 C-118 HAL shipments have carrier_scan_date ≤ 2026-08-31\"}\n```\n", "obj": {"answer": 16, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "16 C-118 HAL shipments have carrier_scan_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-07", "family": "XORG", "rep": 1, "latency_s": 57.26, "prompt_chars": 14108, "raw": "```json\n{\n  \"answer\": 29065,\n  \"definition_id\": \"Order value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden calculation: gross minus discount plus freight.\"\n}\n```\n", "obj": {"answer": 29065, "definition_id": "Order value", "definition_version": 1, "unanswerable": false, "note": "Halden calculation: gross minus discount plus freight."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 61.43, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"active_customers_past_year\",\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"21 customers ordered in 12 months before 2026-07-15\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "active_customers_past_year", "definition_version": null, "unanswerable": false, "note": "21 customers ordered in 12 months before 2026-07-15"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 8.5, "prompt_chars": 12512, "raw": "```json\n{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"Shipments table has no damage or defect tracking field.\"}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Shipments table has no damage or defect tracking field."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 35.34, "prompt_chars": 16506, "raw": "```json\n{\"answer\": 16, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"C-118 shipments with carrier_scan_date on or before 2026-07-31\"}\n```\n", "obj": {"answer": 16, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "C-118 shipments with carrier_scan_date on or before 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 114.48, "prompt_chars": 13987, "raw": "```json\n{\"answer\": 21, \"definition_id\": \"on-time-delivery\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Meridian rule from 1 Feb 2026: revised promised date when populated, else promised date\"}\n```\n", "obj": {"answer": 21, "definition_id": "on-time-delivery", "definition_version": 2, "unanswerable": false, "note": "Meridian rule from 1 Feb 2026: revised promised date when populated, else promised date"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 83.32, "prompt_chars": 16466, "raw": "```json\n{\"answer\": 23, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Applied v2 of on_time (effective 2026-02-01) using revised_promised_date when present\"}\n```\n", "obj": {"answer": 23, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Applied v2 of on_time (effective 2026-02-01) using revised_promised_date when present"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-02", "family": "XORG", "rep": 1, "latency_s": 39.28, "prompt_chars": 14027, "raw": "```json\n{\n  \"answer\": 13,\n  \"definition_id\": \"Delivered shipments - Halden Logistics\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden rule since 2026-01-01: carrier scan date present and on/before cutoff date.\"\n}\n```\n", "obj": {"answer": 13, "definition_id": "Delivered shipments - Halden Logistics", "definition_version": 1, "unanswerable": false, "note": "Halden rule since 2026-01-01: carrier scan date present and on/before cutoff date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 13.67, "prompt_chars": 16969, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Order O066: 2025-12-28 to scan 2026-01-06. Business days: 12/29, 12/30, 12/31, 01/02, 01/05.\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Order O066: 2025-12-28 to scan 2026-01-06. Business days: 12/29, 12/30, 12/31, 01/02, 01/05."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 24.95, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"active_base_past_year\",\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders placed 2025-04-01 to 2026-04-01\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "active_base_past_year", "definition_version": null, "unanswerable": false, "note": "Customers with orders placed 2025-04-01 to 2026-04-01"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-01", "family": "TEMP", "rep": 1, "latency_s": 46.87, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders in past 12 months (2025-02-28 to 2026-02-28)\"\n}\n```\n", "obj": {"answer": 21, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with orders in past 12 months (2025-02-28 to 2026-02-28)"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-02", "family": "XORG", "rep": 1, "latency_s": 31.89, "prompt_chars": 15099, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"C-118 shipments with carrier_scan_date on/before 2026-06-30\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "C-118 shipments with carrier_scan_date on/before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 92.18, "prompt_chars": 16813, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Customer-specific calendars not provided; incomplete holiday data\"\n}\n```\n\n**Reasoning:** The question asks for lateness measured \"under the customer's own working calendar,\" but:\n\n1. **Missing customer calendar data**: Each customer (C001–C024) would have their own working calendar, which is not provided in the data.\n\n2. **Incomplete holiday information**: The available definitions specify Meridian and Halden holidays only for 2026. Many orders span 2024–2025, for which holiday", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Customer-specific calendars not provided; incomplete holiday data"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 23.94, "prompt_chars": 11148, "raw": "```json\n{\n  \"answer\": 37,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date ≤ 2026-05-31\"\n}\n```\n", "obj": {"answer": 37, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with customer_confirm_date ≤ 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 31.74, "prompt_chars": 16512, "raw": "```json\n{\"answer\": 139070, \"definition_id\": \"meridian.def.order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Sum of gross_amount minus discount_amount for non-cancelled orders placed 2026-05-01 to 2026-07-31.\"}\n```\n", "obj": {"answer": 139070, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum of gross_amount minus discount_amount for non-cancelled orders placed 2026-05-01 to 2026-07-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 52.24, "prompt_chars": 15059, "raw": "```json\n{\n  \"answer\": 23,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Delivered shipments (customer_confirm_date present and ≤2026-08-31) assessed against revised_promised_date when present, else promised_date.\"\n}\n```\n", "obj": {"answer": 23, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Delivered shipments (customer_confirm_date present and ≤2026-08-31) assessed against revised_promised_date when present, else promised_date."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-04", "family": "ABST", "rep": 1, "latency_s": 8.31, "prompt_chars": 13962, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer satisfaction score data available in tables.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score data available in tables."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 68.03, "prompt_chars": 13979, "raw": "```json\n{\"answer\": 37, \"definition_id\": \"Delivered shipments\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Counted shipments with customer_confirm_date on/before 2026-05-31\"}\n```\n", "obj": {"answer": 37, "definition_id": "Delivered shipments", "definition_version": 1, "unanswerable": false, "note": "Counted shipments with customer_confirm_date on/before 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 100.18, "prompt_chars": 12722, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Business days from 2025-12-31 to 2026-01-14, excluding weekends and New Year's Day.\"\n}\n```\n", "obj": {"answer": 10, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Business days from 2025-12-31 to 2026-01-14, excluding weekends and New Year's Day."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-09", "family": "TEMP", "rep": 1, "latency_s": 62.31, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Customers with non-cancelled orders in (2025-10-02, 2026-04-01]\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Customers with non-cancelled orders in (2025-10-02, 2026-04-01]"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 28.42, "prompt_chars": 14027, "raw": "```json\n{\n  \"answer\": 14,\n  \"definition_id\": \"Delivered shipments - Halden Logistics rule\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"All C-118 orders are HAL supplier. Counted 14 shipments with carrier_scan_date on/before 2026-06-30.\"\n}\n```\n", "obj": {"answer": 14, "definition_id": "Delivered shipments - Halden Logistics rule", "definition_version": 1, "unanswerable": false, "note": "All C-118 orders are HAL supplier. Counted 14 shipments with carrier_scan_date on/before 2026-06-30."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 141.1, "prompt_chars": 16796, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"12 delivered shipments were on time as of 2026-02-01\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "12 delivered shipments were on time as of 2026-02-01"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 37.96, "prompt_chars": 15051, "raw": "```json\n{\"answer\": 50, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Shipments with customer_confirm_date on or before 2026-08-31\"}\n```\n", "obj": {"answer": 50, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date on or before 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 86.35, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day window from 2026-03-04 to 2026-08-31; excluded cancelled orders\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "Active customers", "definition_version": 2, "unanswerable": false, "note": "180-day window from 2026-03-04 to 2026-08-31; excluded cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 109.79, "prompt_chars": 14035, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Halden delivered (carrier scan ≤ 2026-07-31); on-time v2\"\n}\n```\n\n**Methodology**: Identified 15 C-118 orders with HAL supplier. Of these, 13 had shipments with carrier_scan_date ≤ 2026-07-31 (Halden's delivery rule since 1 Jan 2026). Applied on-time rule effective from 1 Feb 2026: shipments are on-time if carrier_scan_date ≤ revised_promised_date (when present) or ≤ promised_date. Result: 7 shipments met both criteria (O004, O007, O013, O021, O024, O045, O066).\n", "obj": {"answer": 7, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Halden delivered (carrier scan ≤ 2026-07-31); on-time v2"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 52.0, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": \"quarterly_past_year\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders between 2025-02-15 and 2026-02-15.\"\n}\n```\n", "obj": {"answer": 20, "definition_id": "quarterly_past_year", "definition_version": 1, "unanswerable": false, "note": "Customers with orders between 2025-02-15 and 2026-02-15."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 69.2, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered (customer_confirm_date ≤ 2026-08-31) and met promised/revised delivery date.\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered (customer_confirm_date ≤ 2026-08-31) and met promised/revised delivery date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-05", "family": "XORG", "rep": 1, "latency_s": 122.15, "prompt_chars": 15107, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"7 of 13 C-118 HAL shipments delivered by 2026-07-31 were on time\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "7 of 13 C-118 HAL shipments delivered by 2026-07-31 were on time"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 92.86, "prompt_chars": 11277, "raw": "```json\n{\n  \"answer\": 34406,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of invoice amounts for HAL C-118 orders May-July 2026.\"\n}\n```\n\n**Calculation:**\n\nFiltered for non-cancelled C-118 orders (supplier HAL) placed 2026-05-01 to 2026-07-31 inclusive:\n\n- O009 (2026-07-14): Invoice I007 = 5,276\n- O045 (2026-05-10): Invoice I035 = 3,470  \n- O065 (2026-07-21): Invoice I051 = 16,485\n- O072 (2026-05-18): Invoice I057 = 9,175\n\n**Total: 5,276 + 3,470 + 16,485 + 9,175 = 34,406**\n", "obj": {"answer": 34406, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of invoice amounts for HAL C-118 orders May-July 2026."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 98.14, "prompt_chars": 11204, "raw": "```json\n{\n  \"answer\": 8,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Counted HAL-supplied C-118 orders with carrier_scan_date ≤ 2026-06-30 where carrier_scan_date ≤ promised_date (or revised_promised_date if present). 13 delivered; 8 on-time.\"\n}\n```\n", "obj": {"answer": 8, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted HAL-supplied C-118 orders with carrier_scan_date ≤ 2026-06-30 where carrier_scan_date ≤ promised_date (or revised_promised_date if present). 13 delivered; 8 on-time."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-08", "family": "TEMP", "rep": 1, "latency_s": 66.26, "prompt_chars": 15071, "raw": "```json\n{\n  \"answer\": 115288,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Invoices dated 2025-10-01 to 2025-12-31, less returns in same period.\"\n}\n```\n", "obj": {"answer": 115288, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Invoices dated 2025-10-01 to 2025-12-31, less returns in same period."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-03", "family": "COMP", "rep": 1, "latency_s": 121.97, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"active_customers_meridian\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day window rule in force June 1, 2026; window: Dec 4, 2025–Jun 1, 2026\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "active_customers_meridian", "definition_version": 2, "unanswerable": false, "note": "180-day window rule in force June 1, 2026; window: Dec 4, 2025–Jun 1, 2026"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-08", "family": "XORG", "rep": 1, "latency_s": 47.48, "prompt_chars": 14108, "raw": "```json\n{\"answer\": 18220, \"definition_id\": \"order_value\", \"definition_version\": null, \"unanswerable\": false, \"note\": \"Halden formula: gross minus discount plus freight.\"}\n```\n\nQualifying orders (HAL supplier, C-118 contract, date 2026-01-01 to 2026-03-31, non-cancelled):\n- O004 (2026-01-23, invoiced): 7500 - 375 + 150 = 7,275\n- O024 (2026-02-26, invoiced): 12700 - 1905 + 150 = 10,945\n\nTotal: 18,220\n\n(Note: O051 dated 2026-03-19 is excluded as cancelled. Per Order value definition, Halden applies: gross amount − discount amount + freight, with tax excluded.)\n", "obj": {"answer": 18220, "definition_id": "order_value", "definition_version": null, "unanswerable": false, "note": "Halden formula: gross minus discount plus freight."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-02", "family": "COMP", "rep": 1, "latency_s": 71.36, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders 2025-07-15 to 2026-07-15\"\n}\n```\n", "obj": {"answer": 21, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with orders 2025-07-15 to 2026-07-15"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 110.39, "prompt_chars": 11204, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Used carrier_scan_date for delivery; revised_promised_date when available.\"\n}\n```\n", "obj": {"answer": 10, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Used carrier_scan_date for delivery; revised_promised_date when available."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-12", "family": "COMP", "rep": 1, "latency_s": 90.46, "prompt_chars": 12595, "raw": "```json\n{\n  \"answer\": 169538,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Order value = gross - discount + freight + tax for non-cancelled orders only.\"\n}\n```\n", "obj": {"answer": 169538, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Order value = gross - discount + freight + tax for non-cancelled orders only."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-09", "family": "COMP", "rep": 1, "latency_s": 25.65, "prompt_chars": 16788, "raw": "```json\n{\n  \"answer\": 38,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customer confirm date present and ≤ 2026-05-31\"\n}\n```\n", "obj": {"answer": 38, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Customer confirm date present and ≤ 2026-05-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-05", "family": "TEMP", "rep": 1, "latency_s": 60.16, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered by customer_confirm_date ≤ 2026-01-31; on-time vs. revised_promised_date if renegotiated, else promised_date.\"\n}\n```\n", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered by customer_confirm_date ≤ 2026-01-31; on-time vs. revised_promised_date if renegotiated, else promised_date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 34.79, "prompt_chars": 14160, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"business_days\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden calendar: Jan 1, 6 are holidays; Dec 29-31, Jan 2, 5 are business days\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "business_days", "definition_version": 1, "unanswerable": false, "note": "Halden calendar: Jan 1, 6 are holidays; Dec 29-31, Jan 2, 5 are business days"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 18.66, "prompt_chars": 12566, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Customer working calendar data not available in dataset\"\n}\n```\n\n**Explanation**: The question requires calculating delivery delays in business days under each customer's specific working calendar. However, the provided dataset contains no information about:\n- Individual customer locations or regions\n- Each customer's working calendar or business day schedule  \n- The applicable public holiday calendars\n\nWhile the data includes promised dates and customer confirmation dates for s", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Customer working calendar data not available in dataset"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-05", "family": "ABST", "rep": 1, "latency_s": 13.79, "prompt_chars": 11173, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer working calendar data provided; cannot compute business days per customer.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer working calendar data provided; cannot compute business days per customer."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-08", "family": "COMP", "rep": 1, "latency_s": 30.01, "prompt_chars": 16788, "raw": "```json\n{\n  \"answer\": 43,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customer_confirm_date present and on/before 2026-06-30\"\n}\n```\n", "obj": {"answer": 43, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Customer_confirm_date present and on/before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 25.01, "prompt_chars": 11168, "raw": "```json\n{\n  \"answer\": 133443,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Q1 2026 invoices (137,020) minus Q1 returns (3,577)\"\n}\n```\n", "obj": {"answer": 133443, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Q1 2026 invoices (137,020) minus Q1 returns (3,577)"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-09", "family": "COMP", "rep": 1, "latency_s": 23.99, "prompt_chars": 15051, "raw": "```json\n{\n  \"answer\": 38,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date present and on/before 2026-05-31\"\n}\n```\n", "obj": {"answer": 38, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present and on/before 2026-05-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-09", "family": "XORG", "rep": 1, "latency_s": 95.96, "prompt_chars": 11329, "raw": "Looking at the data provided:\n\n**Order O006 details:**\n- Order date: 2025-12-30\n- Supplier: HAL (Halden Logistics)\n- Contract: C-118\n\n**Shipment S005 (for O006):**\n- Carrier scan date: 2026-01-14\n\n**Calculating business days from 2025-12-30 to 2026-01-14:**\n\nFirst, I'll determine the day of the week:\n- 2025-01-01 = Wednesday (baseline)\n- 2025 has 365 days (52 weeks + 1 day), so 2025-12-31 = Wednesday\n- 2025-12-30 = Tuesday\n- 2026-01-14 = Wednesday\n\nCounting weekdays (Monday-Friday) *after* 2025-12-30 through 2026-01-14:\n1. 2025-12-31 (Wed)\n2. 2026-01-02 (Fri) — *excluding 2026-01-01 (Thu, New ", "obj": {"answer": 10, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Weekdays 2025-12-30 to 2026-01-14, excluding 2026-01-01"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 15.57, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with contract_status=open by 2026-03-15\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with contract_status=open by 2026-03-15"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 38.89, "prompt_chars": 16842, "raw": "```json\n{\n  \"answer\": 139070,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"15 orders (May–July 2026), non-cancelled, value = gross - discount\"\n}\n```\n", "obj": {"answer": 139070, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "15 orders (May–July 2026), non-cancelled, value = gross - discount"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 62.31, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Window 180 days: (2026-03-05, 2026-08-31]. 18 unique customers with non-cancelled orders.\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window 180 days: (2026-03-05, 2026-08-31]. 18 unique customers with non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-10", "family": "COMP", "rep": 1, "latency_s": 68.35, "prompt_chars": 16796, "raw": "```json\n{\n  \"answer\": 24,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Internal Meridian reporting; v2 effective 2026-02-01 applies as of question date.\"\n}\n```\n", "obj": {"answer": 24, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Internal Meridian reporting; v2 effective 2026-02-01 applies as of question date."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-09", "family": "TEMP", "rep": 1, "latency_s": 68.96, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Window: 2025-10-02 to 2026-04-01; 15 customers with non-cancelled orders\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window: 2025-10-02 to 2026-04-01; 15 customers with non-cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-10", "family": "COMP", "rep": 1, "latency_s": 105.78, "prompt_chars": 15059, "raw": "```json\n{\"answer\": 23, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"23 of 50 delivered shipments met on-time criteria.\"}\n```\n", "obj": {"answer": 23, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "23 of 50 delivered shipments met on-time criteria."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-05", "family": "ABST", "rep": 1, "latency_s": 73.01, "prompt_chars": 12566, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Business day calculation requires day-of-week data not provided in dataset\"\n}\n```\n\n**Explanation**: The question requires calculating delivery delays in business days (excluding weekends and holidays). While I can identify 30+ orders with customer confirmations and determine calendar day delays, accurately converting calendar days to business days requires knowing:\n\n1. Which dates fall on weekends for each date between 2025-2026\n2. Which dates are public holidays (not provided)", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Business day calculation requires day-of-week data not provided in dataset"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 49.83, "prompt_chars": 11277, "raw": "```json\n{\n  \"answer\": 21624,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of invoice amounts for HAL C-118 orders placed 2026-01-01 to 2026-03-31, non-cancelled\"\n}\n```\n", "obj": {"answer": 21624, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of invoice amounts for HAL C-118 orders placed 2026-01-01 to 2026-03-31, non-cancelled"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 25.73, "prompt_chars": 11202, "raw": "```json\n{\"answer\": 153700, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Sum of gross_amount for non-cancelled orders placed 2026-05-01 to 2026-07-31\"}\n```\n", "obj": {"answer": 153700, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of gross_amount for non-cancelled orders placed 2026-05-01 to 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 47.74, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"17 customers with non-cancelled orders in (2025-12-03, 2026-06-01]\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "17 customers with non-cancelled orders in (2025-12-03, 2026-06-01]"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 24.08, "prompt_chars": 16847, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-301 orders with customer_confirm_date present and ≤2026-08-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "C-301 orders with customer_confirm_date present and ≤2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-05", "family": "TEMP", "rep": 1, "latency_s": 26.5, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"10 delivered shipments were on-time as of 2026-01-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "10 delivered shipments were on-time as of 2026-01-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-12", "family": "XORG", "rep": 1, "latency_s": 56.92, "prompt_chars": 16536, "raw": "```json\n{\n  \"answer\": 58155,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"ZEN orders O016, O019, O020, O025, O026, O036, O069; gross minus discount.\"\n}\n```\n", "obj": {"answer": 58155, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "ZEN orders O016, O019, O020, O025, O026, O036, O069; gross minus discount."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-03", "family": "TEMP", "rep": 1, "latency_s": 46.89, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Window 365 days: (2025-03-15, 2026-03-15], non-cancelled orders only\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Window 365 days: (2025-03-15, 2026-03-15], non-cancelled orders only"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 38.12, "prompt_chars": 16478, "raw": "```json\n{\n  \"answer\": 62903,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Period invoices sum 63690, returns 787, net 62903\"\n}\n```\n", "obj": {"answer": 62903, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Period invoices sum 63690, returns 787, net 62903"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 36.03, "prompt_chars": 14160, "raw": "```json\n{\"answer\": 9, \"definition_id\": \"Business days\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"9 business days from 2025-12-30 to 2026-01-14 using Halden 2026 calendar\"}\n```\n", "obj": {"answer": 9, "definition_id": "Business days", "definition_version": 1, "unanswerable": false, "note": "9 business days from 2025-12-30 to 2026-01-14 using Halden 2026 calendar"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-08", "family": "TEMP", "rep": 1, "latency_s": 53.82, "prompt_chars": 16478, "raw": "```json\n{\n  \"answer\": 115288,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"13 invoices dated in period; 1 return; v1 applies through 2026-04-30\"\n}\n```\n", "obj": {"answer": 115288, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "13 invoices dated in period; 1 return; v1 applies through 2026-04-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-09", "family": "XORG", "rep": 1, "latency_s": 17.37, "prompt_chars": 16969, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"9 weekdays excluding Halden holidays 2026-01-01 and 2026-01-06\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "9 weekdays excluding Halden holidays 2026-01-01 and 2026-01-06"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-09", "family": "COMP", "rep": 1, "latency_s": 23.95, "prompt_chars": 13979, "raw": "```json\n{\n  \"answer\": 37,\n  \"definition_id\": \"Delivered shipments\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Meridian: customer_confirm_date present and ≤ 2026-05-31\"\n}\n```\n", "obj": {"answer": 37, "definition_id": "Delivered shipments", "definition_version": 1, "unanswerable": false, "note": "Meridian: customer_confirm_date present and ≤ 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-01", "family": "XORG", "rep": 1, "latency_s": 73.0, "prompt_chars": 16836, "raw": "```json\n{\n  \"answer\": 19,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"19 of 21 C-118 orders have shipments with carrier_scan_date ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 19, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "19 of 21 C-118 orders have shipments with carrier_scan_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-05", "family": "XORG", "rep": 1, "latency_s": 167.57, "prompt_chars": 14035, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"on-time-delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden logistics carrier scan date rule applied.\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "on-time-delivery", "definition_version": 1, "unanswerable": false, "note": "Halden logistics carrier scan date rule applied."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 35.25, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 22,\n  \"definition_id\": \"customers_with_orders_past_12_months\",\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders placed between 2025-06-01 and 2026-06-01 inclusive.\"\n}\n```\n", "obj": {"answer": 22, "definition_id": "customers_with_orders_past_12_months", "definition_version": null, "unanswerable": false, "note": "Customers with orders placed between 2025-06-01 and 2026-06-01 inclusive."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-06", "family": "COMP", "rep": 1, "latency_s": 46.93, "prompt_chars": 16478, "raw": "```json\n{\n  \"answer\": 62903,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Invoices 2026-07-01 to 2026-08-31: 7 invoices, revenue 63690, returns 787\"\n}\n```\n", "obj": {"answer": 62903, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Invoices 2026-07-01 to 2026-08-31: 7 invoices, revenue 63690, returns 787"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-03", "family": "ABST", "rep": 1, "latency_s": 8.79, "prompt_chars": 15022, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Shipments table lacks damage status field.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Shipments table lacks damage status field."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-03", "family": "TEMP", "rep": 1, "latency_s": 12.24, "prompt_chars": 11151, "raw": "```json\n{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active customers = signed up by date with open contract_status\"}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active customers = signed up by date with open contract_status"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 94.02, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered by customer_confirm_date ≤ 2026-01-31; on-time = customer_confirm_date ≤ agreed_promised_date\"\n}\n```\n", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered by customer_confirm_date ≤ 2026-01-31; on-time = customer_confirm_date ≤ agreed_promised_date"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-10", "family": "COMP", "rep": 1, "latency_s": 72.4, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 23,\n  \"definition_id\": \"meridian_on_time_performance\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Used revised_promised_date when available; compared customer_confirm_date to commitment.\"\n}\n```\n", "obj": {"answer": 23, "definition_id": "meridian_on_time_performance", "definition_version": 1, "unanswerable": false, "note": "Used revised_promised_date when available; compared customer_confirm_date to commitment."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 118.13, "prompt_chars": 12561, "raw": "```json\n{\n  \"answer\": 141511,\n  \"definition_id\": \"revenue_invoiced_minus_returns\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Invoiced Q4 2025 amount minus returns in period.\"\n}\n```\n", "obj": {"answer": 141511, "definition_id": "revenue_invoiced_minus_returns", "definition_version": 1, "unanswerable": false, "note": "Invoiced Q4 2025 amount minus returns in period."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-07", "family": "XORG", "rep": 0, "latency_s": 59.75, "prompt_chars": 14108, "raw": "```json\n{\"answer\": 29065, \"definition_id\": \"Order value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Halden formula: gross - discount + freight.\"}\n```\n", "obj": {"answer": 29065, "definition_id": "Order value", "definition_version": 1, "unanswerable": false, "note": "Halden formula: gross - discount + freight."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-12", "family": "COMP", "rep": 1, "latency_s": 22.54, "prompt_chars": 16512, "raw": "```json\n{\n  \"answer\": 139070,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"15 non-cancelled orders 2026-05-01 to 2026-07-31; gross minus discount\"\n}\n```\n", "obj": {"answer": 139070, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "15 non-cancelled orders 2026-05-01 to 2026-07-31; gross minus discount"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-02", "family": "COMP", "rep": 1, "latency_s": 65.73, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"180-day window [2026-01-17, 2026-07-15]; non-cancelled orders only.\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "Active customers", "definition_version": 1, "unanswerable": false, "note": "180-day window [2026-01-17, 2026-07-15]; non-cancelled orders only."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 45.68, "prompt_chars": 16788, "raw": "```json\n{\n  \"answer\": 37,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date present and on/before 2026-05-31\"\n}\n```\n", "obj": {"answer": 37, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present and on/before 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 54.24, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date ≤ promised_date (or revised_promised_date if present)\"\n}\n```\n", "obj": {"answer": 21, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with customer_confirm_date ≤ promised_date (or revised_promised_date if present)"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-07", "family": "COMP", "rep": 1, "latency_s": 16.82, "prompt_chars": 16788, "raw": "```json\n{\"answer\": 51, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Count of shipments with customer_confirm_date on/before 2026-08-31\"}\n```\n", "obj": {"answer": 51, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Count of shipments with customer_confirm_date on/before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 79.13, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Window 365 days: (2025-02-28, 2026-02-28]. Non-cancelled orders.\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Window 365 days: (2025-02-28, 2026-02-28]. Non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-04", "family": "XORG", "rep": 1, "latency_s": 34.08, "prompt_chars": 14035, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Halden delivery rule (carrier scan date) + on-time rule from 1 Feb 2026\"\n}\n```\n", "obj": {"answer": 9, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Halden delivery rule (carrier scan date) + on-time rule from 1 Feb 2026"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-02", "family": "COMP", "rep": 1, "latency_s": 11.89, "prompt_chars": 11151, "raw": "```json\n{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active = contract_status open; all customers signed up by cutoff\"}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = contract_status open; all customers signed up by cutoff"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-11", "family": "XORG", "rep": 1, "latency_s": 21.12, "prompt_chars": 15110, "raw": "```json\n{\"answer\": 11, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"11 C-301 shipments have customer_confirm_date on/before 2026-08-31\"}\n```\n", "obj": {"answer": 11, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "11 C-301 shipments have customer_confirm_date on/before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 25.21, "prompt_chars": 12619, "raw": "```json\n{\n  \"answer\": 62700,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of gross_amount for non-cancelled ZEN orders placed 2026-06-01 to 2026-08-31.\"\n}\n```\n", "obj": {"answer": 62700, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of gross_amount for non-cancelled ZEN orders placed 2026-06-01 to 2026-08-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 47.73, "prompt_chars": 15051, "raw": "```json\n{\n  \"answer\": 37,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"37 shipments with customer_confirm_date on/before 2026-05-31\"\n}\n```\n", "obj": {"answer": 37, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "37 shipments with customer_confirm_date on/before 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 162.13, "prompt_chars": 13982, "raw": "Looking at this question, I need to determine how many active customers existed as of 2026-01-31 using the definition provided.\n\n**Understanding the Definition:**\n- A customer is active if they have at least one order with status ≠ \"cancelled\" and order_date within the active window\n- For dates between 1 January 2025 and 31 March 2026, the window is 365 days\n- Since 2026-01-31 falls within this period, I use a 365-day window\n- The window ending on 2026-01-31: since 2025 has exactly 365 days, 365 days before 2026-01-31 is 2025-01-31\n- Therefore the window is [2025-01-31, 2026-01-31] inclusive\n\n", "obj": {"answer": 16, "definition_id": "active_customers", "definition_version": 1, "unanswerable": false, "note": "365-day window (1 Jan 2025 - 31 Mar 2026 period)"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 70.17, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"365-day window active orders: 2025-02-01 through 2026-01-31\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window active orders: 2025-02-01 through 2026-01-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 36.28, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Active customers with non-cancelled orders in 180 days prior to 2026-04-01.\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Active customers with non-cancelled orders in 180 days prior to 2026-04-01."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 213.99, "prompt_chars": 16483, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"meridian.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"18 delivered orders exceeded 2-business-day lateness using Meridian calendar.\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "meridian.def.business_day", "definition_version": 1, "unanswerable": false, "note": "18 delivered orders exceeded 2-business-day lateness using Meridian calendar."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-01", "family": "ABST", "rep": 1, "latency_s": 98.14, "prompt_chars": 12518, "raw": "```json\n{\n  \"answer\": 2,\n  \"definition_id\": \"order_based_churn\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Q1→Q2 active base decline. Customers C001, C007 ordered in Q1 but not Q2.\"\n}\n```\n", "obj": {"answer": 2, "definition_id": "order_based_churn", "definition_version": 1, "unanswerable": false, "note": "Q1→Q2 active base decline. Customers C001, C007 ordered in Q1 but not Q2."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 19.4, "prompt_chars": 15071, "raw": "```json\n{\n  \"answer\": 55463,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Invoices dated 2026-07-01 to 2026-08-31: I026, I029, I048, I051, I054. Calculated (gross - discount + freight) for each. Subtracted return R004 (2026-07-11).\"\n}\n```\n", "obj": {"answer": 55463, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Invoices dated 2026-07-01 to 2026-08-31: I026, I029, I048, I051, I054. Calculated (gross - discount + freight) for each. Subtracted return R004 (2026-07-11)."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-11", "family": "XORG", "rep": 1, "latency_s": 22.76, "prompt_chars": 16517, "raw": "```json\n{\n  \"answer\": 11,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-301 HAL orders with customer_confirm_date on/before 2026-08-31\"\n}\n```\n", "obj": {"answer": 11, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "C-301 HAL orders with customer_confirm_date on/before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-01", "family": "COMP", "rep": 1, "latency_s": 40.78, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"18 customers with non-cancelled orders in 180-day window\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "18 customers with non-cancelled orders in 180-day window"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 30.26, "prompt_chars": 16836, "raw": "```json\n{\"answer\": 15, \"definition_id\": \"halden.def.delivered\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Fed.C-118.delivered applies; halden def requires carrier_scan_date present and on/before as-of date\"}\n```\n", "obj": {"answer": 15, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "Fed.C-118.delivered applies; halden def requires carrier_scan_date present and on/before as-of date"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-02", "family": "TEMP", "rep": 1, "latency_s": 87.19, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 11,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Window 365 days: (2025-01-31, 2026-01-31]. Qualifying customers: C001, C003, C005, C008, C011, C013, C017, C021, C022, C023, C024\"\n}\n```\n", "obj": {"answer": 11, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Window 365 days: (2025-01-31, 2026-01-31]. Qualifying customers: C001, C003, C005, C008, C011, C013, C017, C021, C022, C023, C024"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-08", "family": "COMP", "rep": 1, "latency_s": 53.7, "prompt_chars": 13979, "raw": "```json\n{\"answer\": 43, \"definition_id\": \"Delivered shipments\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Customer confirmation date present and ≤ 2026-06-30\"}\n```\n", "obj": {"answer": 43, "definition_id": "Delivered shipments", "definition_version": 1, "unanswerable": false, "note": "Customer confirmation date present and ≤ 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 100.34, "prompt_chars": 15024, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"5 customers have non-cancelled orders in window (2026-07-05, 2026-09-06]\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "5 customers have non-cancelled orders in window (2026-07-05, 2026-09-06]"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-04", "family": "COMP", "rep": 1, "latency_s": 49.34, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"quarterly_active_base\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders May 2025-May 2026\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "quarterly_active_base", "definition_version": 1, "unanswerable": false, "note": "Customers with orders May 2025-May 2026"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 51.39, "prompt_chars": 16517, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"10 C-301 HAL shipments delivered as of 2026-08-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "10 C-301 HAL shipments delivered as of 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-05", "family": "TEMP", "rep": 1, "latency_s": 51.05, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered on/before 2026-01-31 with confirmed date ≤ promised/revised promised date\"\n}\n```\n", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered on/before 2026-01-31 with confirmed date ≤ promised/revised promised date"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-12", "family": "XORG", "rep": 1, "latency_s": 27.7, "prompt_chars": 14057, "raw": "```json\n{\n  \"answer\": 58155,\n  \"definition_id\": \"Order value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"7 non-cancelled ZEN orders 2026-06-01 to 2026-08-31. Gross minus discount.\"\n}\n```\n", "obj": {"answer": 58155, "definition_id": "Order value", "definition_version": 1, "unanswerable": false, "note": "7 non-cancelled ZEN orders 2026-06-01 to 2026-08-31. Gross minus discount."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 58.55, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Version 1 active (365-day window), period: 2025-03-01 to 2026-02-28\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Version 1 active (365-day window), period: 2025-03-01 to 2026-02-28"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-12", "family": "XORG", "rep": 1, "latency_s": 22.49, "prompt_chars": 16866, "raw": "```json\n{\n  \"answer\": 58155,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Non-cancelled ZEN orders 2026-06-01 to 2026-08-31\"\n}\n```\n", "obj": {"answer": 58155, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Non-cancelled ZEN orders 2026-06-01 to 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-06", "family": "TEMP", "rep": 1, "latency_s": 37.58, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered = customer_confirm_date ≤ date. On-time = met promised or revised promised date.\"\n}\n```\n", "obj": {"answer": 9, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date ≤ date. On-time = met promised or revised promised date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 9.46, "prompt_chars": 16441, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Customer satisfaction score not present in data tables\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Customer satisfaction score not present in data tables"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-07", "family": "TEMP", "rep": 1, "latency_s": 59.95, "prompt_chars": 16478, "raw": "```json\n{\"answer\": 109343, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Invoices dated Q1 2026 minus Q1 returns\"}\n```\n", "obj": {"answer": 109343, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Invoices dated Q1 2026 minus Q1 returns"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-06", "family": "COMP", "rep": 1, "latency_s": 67.01, "prompt_chars": 11168, "raw": "```json\n{\"answer\": 74641, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Invoiced revenue (75428) minus returns (787) recognized in period.\"}\n```\n", "obj": {"answer": 74641, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Invoiced revenue (75428) minus returns (787) recognized in period."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-01", "family": "COMP", "rep": 1, "latency_s": 60.07, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day window: 2026-03-05 to 2026-08-31. 18 of 24 customers had non-cancelled orders.\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day window: 2026-03-05 to 2026-08-31. 18 of 24 customers had non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-08", "family": "XORG", "rep": 1, "latency_s": 70.76, "prompt_chars": 12670, "raw": "```json\n{\n  \"answer\": 21624,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of invoice amounts for HAL C-118 orders placed 2026-01-01 to 2026-03-31.\"\n}\n```\n", "obj": {"answer": 21624, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of invoice amounts for HAL C-118 orders placed 2026-01-01 to 2026-03-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-06", "family": "TEMP", "rep": 1, "latency_s": 51.31, "prompt_chars": 16796, "raw": "```json\n{\"answer\": 7, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Delivered shipments where customer_confirm_date ≤ promised_date as of 2025-12-31.\"}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Delivered shipments where customer_confirm_date ≤ promised_date as of 2025-12-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-09", "family": "XORG", "rep": 1, "latency_s": 28.03, "prompt_chars": 15232, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Counted weekdays excluding Halden holidays 2026-01-01, 2026-01-06 from 2025-12-31 through 2026-01-14.\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Counted weekdays excluding Halden holidays 2026-01-01, 2026-01-06 from 2025-12-31 through 2026-01-14."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 111.91, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"on-time_delivery\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"21 delivered shipments on-time per revised rule from 1 Feb 2026\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "on-time_delivery", "definition_version": 2, "unanswerable": false, "note": "21 delivered shipments on-time per revised rule from 1 Feb 2026"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-03", "family": "ABST", "rep": 1, "latency_s": 9.61, "prompt_chars": 12512, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No damage tracking field in shipments or returns data.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damage tracking field in shipments or returns data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 57.84, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 16,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day window from 2025-11-16. Customers with non-cancelled orders in period.\"\n}\n```\n", "obj": {"answer": 16, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day window from 2025-11-16. Customers with non-cancelled orders in period."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-02", "family": "TEMP", "rep": 1, "latency_s": 46.56, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 19,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"365-day window 2025-02-01 to 2026-01-31, status not cancelled\"\n}\n```\n", "obj": {"answer": 19, "definition_id": "Active customers", "definition_version": 1, "unanswerable": false, "note": "365-day window 2025-02-01 to 2026-01-31, status not cancelled"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-05", "family": "TEMP", "rep": 1, "latency_s": 45.48, "prompt_chars": 16796, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"10 delivered shipments were on time as of 2026-01-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "10 delivered shipments were on time as of 2026-01-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 123.46, "prompt_chars": 15107, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Applied halden.delivered v2 and meridian.on_time v2\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Applied halden.delivered v2 and meridian.on_time v2"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-01", "family": "XORG", "rep": 1, "latency_s": 35.18, "prompt_chars": 16506, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"All C-118 HAL shipments have carrier_scan_date ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "All C-118 HAL shipments have carrier_scan_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-06", "family": "ABST", "rep": 1, "latency_s": 15.03, "prompt_chars": 11121, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active customers = contract_status='open'; 4 months ahead extrapolation assumes no status changes.\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active customers = contract_status='open'; 4 months ahead extrapolation assumes no status changes."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-09", "family": "COMP", "rep": 1, "latency_s": 64.71, "prompt_chars": 11148, "raw": "```json\n{\n  \"answer\": 37,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Counted shipments with non-null customer_confirm_date ≤ 2026-05-31\"\n}\n```\n", "obj": {"answer": 37, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted shipments with non-null customer_confirm_date ≤ 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-03", "family": "ABST", "rep": 1, "latency_s": 9.94, "prompt_chars": 13950, "raw": "```json\n{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No data field indicates shipment damage status.\"}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No data field indicates shipment damage status."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-07", "family": "COMP", "rep": 1, "latency_s": 43.17, "prompt_chars": 15051, "raw": "```json\n{\"answer\": 50, \"definition_id\": \"meridian.def.delivered\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Shipments with customer_confirm_date present and on/before 2026-08-31\"}\n```\n", "obj": {"answer": 50, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present and on/before 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 77.07, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"active_customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"180-day window from 2026-01-17 to 2026-07-15; excluded cancelled orders\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "active_customers", "definition_version": 1, "unanswerable": false, "note": "180-day window from 2026-01-17 to 2026-07-15; excluded cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-10", "family": "COMP", "rep": 1, "latency_s": 48.95, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"Meridian On-Time Delivery\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Applied new rule (Feb 2026+) and old rule (<Feb 2026) for deliveries as of 2026-08-31\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "Meridian On-Time Delivery", "definition_version": 2, "unanswerable": false, "note": "Applied new rule (Feb 2026+) and old rule (<Feb 2026) for deliveries as of 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-09", "family": "TEMP", "rep": 1, "latency_s": 50.38, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"active_past_year_orders\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders placed 2025-04-01 to 2026-04-01 inclusive\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "active_past_year_orders", "definition_version": 1, "unanswerable": false, "note": "Customers with orders placed 2025-04-01 to 2026-04-01 inclusive"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-04", "family": "XORG", "rep": 1, "latency_s": 105.96, "prompt_chars": 15107, "raw": "```json\n{\"answer\": 10, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Applied Halden delivery definition; carrier_scan date\"}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Applied Halden delivery definition; carrier_scan date"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 55.82, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 11,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Delivered shipments with customer_confirm_date ≤ 2026-01-31 and delivery date ≤ promised_date.\"\n}\n```\n", "obj": {"answer": 11, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Delivered shipments with customer_confirm_date ≤ 2026-01-31 and delivery date ≤ promised_date."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-02", "family": "ABST", "rep": 1, "latency_s": 23.25, "prompt_chars": 11104, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Gross margin requires Cost of Goods Sold; only revenue-side data available.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Gross margin requires Cost of Goods Sold; only revenue-side data available."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 48.48, "prompt_chars": 15059, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"10 shipments delivered by 2026-01-31 met on-time criteria\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "10 shipments delivered by 2026-01-31 met on-time criteria"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 23.51, "prompt_chars": 13935, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No cost of goods sold data provided; gross margin undefined in handbook.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No cost of goods sold data provided; gross margin undefined in handbook."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-05", "family": "COMP", "rep": 1, "latency_s": 30.64, "prompt_chars": 13999, "raw": "```json\n{\"answer\": 104670, \"definition_id\": \"net_revenue_meridian\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Invoices dated 2026-05-01 to 2026-06-30 inclusive, freight included from 1 May 2026, minus returns in period.\"}\n```\n", "obj": {"answer": 104670, "definition_id": "net_revenue_meridian", "definition_version": 2, "unanswerable": false, "note": "Invoices dated 2026-05-01 to 2026-06-30 inclusive, freight included from 1 May 2026, minus returns in period."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 56.62, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 16,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Interval: 2025-11-15 to 2026-05-15, window_days=180\"\n}\n```\n", "obj": {"answer": 16, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Interval: 2025-11-15 to 2026-05-15, window_days=180"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 27.54, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Delivered shipments with customer_confirm_date <= promised_date\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Delivered shipments with customer_confirm_date <= promised_date"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-06", "family": "COMP", "rep": 1, "latency_s": 117.46, "prompt_chars": 12561, "raw": "```json\n{\"answer\": 74641, \"definition_id\": \"revenue-period-invoiced-minus-returns-period\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Invoices issued 2026-07-01 to 2026-08-31 minus returns recorded in period.\"}\n```\n", "obj": {"answer": 74641, "definition_id": "revenue-period-invoiced-minus-returns-period", "definition_version": 1, "unanswerable": false, "note": "Invoices issued 2026-07-01 to 2026-08-31 minus returns recorded in period."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 123.27, "prompt_chars": 14004, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Cannot calculate business days without calendar day-of-week functions\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Cannot calculate business days without calendar day-of-week functions"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 74.67, "prompt_chars": 11104, "raw": "```json\n{\n  \"answer\": 97,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Gross margin: (net revenue - freight) / net revenue × 100. July 2026 orders: O009, O031, O036, O065.\"\n}\n```\n", "obj": {"answer": 97, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Gross margin: (net revenue - freight) / net revenue × 100. July 2026 orders: O009, O031, O036, O065."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-02", "family": "XORG", "rep": 1, "latency_s": 33.92, "prompt_chars": 16506, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"C-118 HAL shipments with carrier_scan_date ≤ 2026-06-30\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "C-118 HAL shipments with carrier_scan_date ≤ 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 72.63, "prompt_chars": 12600, "raw": "```json\n{\"answer\": 11, \"definition_id\": \"delivery_customer_confirm\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"C-301 HAL shipments with customer_confirm_date on/before 2026-08-31\"}\n```\n", "obj": {"answer": 11, "definition_id": "delivery_customer_confirm", "definition_version": 1, "unanswerable": false, "note": "C-301 HAL shipments with customer_confirm_date on/before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 35.28, "prompt_chars": 16917, "raw": "```json\n{\n  \"answer\": 18220,\n  \"definition_id\": \"halden.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-118 orders 2026-01-01 to 2026-03-31, non-cancelled: O004 + O024\"\n}\n```\n", "obj": {"answer": 18220, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "C-118 orders 2026-01-01 to 2026-03-31, non-cancelled: O004 + O024"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 51.64, "prompt_chars": 16506, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"18 C-118 HAL shipments have carrier_scan_date ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "18 C-118 HAL shipments have carrier_scan_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-01", "family": "TEMP", "rep": 1, "latency_s": 129.48, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"365-day window applies on 2026-02-28; 21 customers had non-cancelled orders placed from 2025-02-28 through 2026-02-28\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "Active customers", "definition_version": 1, "unanswerable": false, "note": "365-day window applies on 2026-02-28; 21 customers had non-cancelled orders placed from 2025-02-28 through 2026-02-28"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-02", "family": "XORG", "rep": 1, "latency_s": 120.8, "prompt_chars": 11196, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Counted C-118 HAL shipments with confirmed customer delivery by 2026-06-30\"\n}\n```\n", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted C-118 HAL shipments with confirmed customer delivery by 2026-06-30"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-12", "family": "COMP", "rep": 1, "latency_s": 87.74, "prompt_chars": 11202, "raw": "```json\n{\n  \"answer\": 169538,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Order value = gross_amount - discount_amount + freight_amount + tax_amount. 15 non-cancelled orders in date range.\"\n}\n```\n", "obj": {"answer": 169538, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Order value = gross_amount - discount_amount + freight_amount + tax_amount. 15 non-cancelled orders in date range."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-03", "family": "ABST", "rep": 1, "latency_s": 8.44, "prompt_chars": 16429, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No damage tracking field exists in shipments data\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damage tracking field exists in shipments data"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 70.87, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"365-day window: (2025-02-28, 2026-02-28]; non-cancelled orders\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window: (2025-02-28, 2026-02-28]; non-cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-10", "family": "XORG", "rep": 1, "latency_s": 13.18, "prompt_chars": 12722, "raw": "```json\n{\"answer\": 6, \"definition_id\": \"business_days_standard\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Excludes weekends and 2026-01-01 (New Year's Day)\"}\n```\n\n**Calculation:**\n- Order date: 2025-12-28 (Sunday)\n- Carrier scan date: 2026-01-06 (Tuesday)\n- Business days after order date through scan date:\n  1. 2025-12-29 (Monday)\n  2. 2025-12-30 (Tuesday)\n  3. 2025-12-31 (Wednesday)\n  4. 2026-01-02 (Friday) — skips 2026-01-01 (public holiday)\n  5. 2026-01-05 (Monday)\n  6. 2026-01-06 (Tuesday) — included\n", "obj": {"answer": 6, "definition_id": "business_days_standard", "definition_version": 1, "unanswerable": false, "note": "Excludes weekends and 2026-01-01 (New Year's Day)"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-07", "family": "XORG", "rep": 1, "latency_s": 41.75, "prompt_chars": 12670, "raw": "```json\n{\n  \"answer\": 34406,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Invoiced amount for 4 non-cancelled HAL C-118 orders placed 2026-05-01 to 2026-07-31\"\n}\n```\n", "obj": {"answer": 34406, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Invoiced amount for 4 non-cancelled HAL C-118 orders placed 2026-05-01 to 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 52.0, "prompt_chars": 15054, "raw": "```json\n{\"answer\": 15, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"180-day lookback window from 2026-04-01: (2025-10-03, 2026-04-01]\"}\n```\n", "obj": {"answer": 15, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day lookback window from 2026-04-01: (2025-10-03, 2026-04-01]"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 49.7, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"past_year_orders\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"21 customers ordered in past year as of 2026-02-28\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "past_year_orders", "definition_version": 1, "unanswerable": false, "note": "21 customers ordered in past year as of 2026-02-28"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-03", "family": "XORG", "rep": 1, "latency_s": 32.36, "prompt_chars": 16836, "raw": "```json\n{\n  \"answer\": 13,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"C-118 shipments with carrier_scan_date on/before 2026-07-31\"\n}\n```\n", "obj": {"answer": 13, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "C-118 shipments with carrier_scan_date on/before 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-02", "family": "TEMP", "rep": 1, "latency_s": 51.45, "prompt_chars": 16461, "raw": "```json\n{\"answer\": 19, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Window: (2025-01-31, 2026-01-31], non-cancelled orders only.\"}\n```\n", "obj": {"answer": 19, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Window: (2025-01-31, 2026-01-31], non-cancelled orders only."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 102.4, "prompt_chars": 14035, "raw": "```json\n{\n  \"answer\": 11,\n  \"definition_id\": \"On-time delivery\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Halden delivery rule (carrier scan date). Feb 2026+ on-time definition applied.\"\n}\n```\n", "obj": {"answer": 11, "definition_id": "On-time delivery", "definition_version": 2, "unanswerable": false, "note": "Halden delivery rule (carrier scan date). Feb 2026+ on-time definition applied."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-04", "family": "XORG", "rep": 1, "latency_s": 60.28, "prompt_chars": 16844, "raw": "```json\n{\"answer\": 11, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"C-118 HAL orders with delivered shipments: 18 total, 11 on time by revised date\"}\n```\n", "obj": {"answer": 11, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "C-118 HAL orders with delivered shipments: 18 total, 11 on time by revised date"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-03", "family": "COMP", "rep": 1, "latency_s": 65.36, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"17 customers with non-cancelled orders in 180-day window\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "17 customers with non-cancelled orders in 180-day window"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 51.0, "prompt_chars": 16791, "raw": "```json\n{\"answer\": 17, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Version 2 effective from 2026-04-01, 180-day window applied\"}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Version 2 effective from 2026-04-01, 180-day window applied"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-06", "family": "COMP", "rep": 1, "latency_s": 32.97, "prompt_chars": 15071, "raw": "```json\n{\n  \"answer\": 62903,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Sum of (gross - discount + freight) for invoices dated 2026-07-01 to 2026-08-31, minus returns in period.\"\n}\n```\n", "obj": {"answer": 62903, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Sum of (gross - discount + freight) for invoices dated 2026-07-01 to 2026-08-31, minus returns in period."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-05", "family": "ABST", "rep": 0, "latency_s": 224.62, "prompt_chars": 15076, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Orders late >2 business days using Meridian calendar\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Orders late >2 business days using Meridian calendar"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 36.02, "prompt_chars": 16458, "raw": "```json\n{\n  \"answer\": 37,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date on or before 2026-05-31.\"\n}\n```\n", "obj": {"answer": 37, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date on or before 2026-05-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 66.02, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 19,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customers with non-cancelled orders dated between 2025-01-31 and 2026-01-31.\"\n}\n```\n", "obj": {"answer": 19, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Customers with non-cancelled orders dated between 2025-01-31 and 2026-01-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 89.33, "prompt_chars": 11207, "raw": "```json\n{\"answer\": 10, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Counted C-301 HAL shipments with customer_confirm_date ≤ 2026-08-31\"}\n```\n", "obj": {"answer": 10, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted C-301 HAL shipments with customer_confirm_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-11", "family": "COMP", "rep": 1, "latency_s": 74.38, "prompt_chars": 16796, "raw": "```json\n{\"answer\": 21, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"21 of 45 delivered shipments were on-time per v2\"}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "21 of 45 delivered shipments were on-time per v2"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-05", "family": "COMP", "rep": 1, "latency_s": 88.68, "prompt_chars": 12561, "raw": "```json\n{\n  \"answer\": 91190,\n  \"definition_id\": \"net_revenue_with_returns\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Invoice_date period. Net revenue = (gross - discount) + freight - returns.\"\n}\n```\n", "obj": {"answer": 91190, "definition_id": "net_revenue_with_returns", "definition_version": 1, "unanswerable": false, "note": "Invoice_date period. Net revenue = (gross - discount) + freight - returns."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-04", "family": "XORG", "rep": 1, "latency_s": 45.68, "prompt_chars": 12597, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": \"context_on_time_delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-118 HAL shipments delivered and on-time by 2026-08-31\"\n}\n```\n\n**Calculation details:**\n\nC-118 orders with HAL supplier that were delivered (customer_confirm_date set) by 2026-08-31:\n\n**On-time (6 shipments):**\n- O004: delivered 2026-02-06 vs revised promised 2026-02-08 ✓\n- O013: delivered 2025-08-17 vs promised 2025-08-21 ✓\n- O021: delivered 2025-08-28 vs promised 2025-08-30 ✓\n- O024: delivered 2026-03-14 vs promised 2026-03-15 ✓\n- O045: delivered 2026-05-20 ", "obj": {"answer": 6, "definition_id": "context_on_time_delivery", "definition_version": 1, "unanswerable": false, "note": "C-118 HAL shipments delivered and on-time by 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 39.11, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"meridian_on_time_delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Delivered = confirmed by customer by 2026-02-01; on-time = confirmed ≤ revised_promised_date (or promised_date if not revised)\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "meridian_on_time_delivery", "definition_version": 1, "unanswerable": false, "note": "Delivered = confirmed by customer by 2026-02-01; on-time = confirmed ≤ revised_promised_date (or promised_date if not revised)"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-05", "family": "COMP", "rep": 1, "latency_s": 33.46, "prompt_chars": 16808, "raw": "```json\n{\n  \"answer\": 104670,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Invoices dated 2026-05-01 to 2026-06-30; minus returns same period.\"\n}\n```\n", "obj": {"answer": 104670, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Invoices dated 2026-05-01 to 2026-06-30; minus returns same period."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-01", "family": "TEMP", "rep": 0, "latency_s": 13.06, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with signup_date ≤ 2026-02-28 and contract_status=open\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with signup_date ≤ 2026-02-28 and contract_status=open"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 14.66, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active customers: contract_status='open' as of signup date ≤ 2026-04-01\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active customers: contract_status='open' as of signup date ≤ 2026-04-01"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-02", "family": "ABST", "rep": 1, "latency_s": 15.67, "prompt_chars": 15007, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No gross margin definition in catalogue; COGS data unavailable\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No gross margin definition in catalogue; COGS data unavailable"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-05", "family": "ABST", "rep": 1, "latency_s": 47.61, "prompt_chars": 16483, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer business day definition available in catalogue\"\n}\n```\n\n**Rationale:** The question refers to \"the customer's own working calendar,\" but the definition catalogue provides only two business day definitions:\n- `meridian.def.business_day` (for Meridian)\n- `halden.def.business_day` (for Halden Logistics)\n\nNeither defines a customer-specific or customer-agnostic working calendar. Without a definition establishing which business days apply to individual customers, the late", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer business day definition available in catalogue"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 42.18, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Period: 2025-03-16 to 2026-03-15, window_days=365\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Period: 2025-03-16 to 2026-03-15, window_days=365"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-03", "family": "COMP", "rep": 1, "latency_s": 89.6, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Applied v2 (180-day window). Period: 2025-12-05 to 2026-06-01.\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Applied v2 (180-day window). Period: 2025-12-05 to 2026-06-01."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-08", "family": "TEMP", "rep": 1, "latency_s": 37.55, "prompt_chars": 16808, "raw": "```json\n{\n  \"answer\": 115288,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Sum of (gross - discount) for 13 invoices dated 2025-10-01 to 2025-12-31 minus one return (R003, 6697).\"\n}\n```\n", "obj": {"answer": 115288, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum of (gross - discount) for 13 invoices dated 2025-10-01 to 2025-12-31 minus one return (R003, 6697)."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 113.5, "prompt_chars": 13982, "raw": "```json\n{\"answer\": 15, \"definition_id\": \"Active customers\", \"definition_version\": null, \"unanswerable\": false, \"note\": \"180-day window from Oct 2025 to Apr 2026\"}\n```\n", "obj": {"answer": 15, "definition_id": "Active customers", "definition_version": null, "unanswerable": false, "note": "180-day window from Oct 2025 to Apr 2026"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 63.87, "prompt_chars": 13999, "raw": "```json\n{\n  \"answer\": 115288,\n  \"definition_id\": \"Net revenue\",\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Invoices dated 2025-10-01 to 2025-12-31, gross minus discount, less returns.\"\n}\n```\n", "obj": {"answer": 115288, "definition_id": "Net revenue", "definition_version": null, "unanswerable": false, "note": "Invoices dated 2025-10-01 to 2025-12-31, gross minus discount, less returns."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-02", "family": "XORG", "rep": 1, "latency_s": 22.74, "prompt_chars": 12589, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"delivery_receipt\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-118 HAL shipments with customer confirmation by 2026-06-30\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "delivery_receipt", "definition_version": 1, "unanswerable": false, "note": "C-118 HAL shipments with customer confirmation by 2026-06-30"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-05", "family": "XORG", "rep": 1, "latency_s": 97.57, "prompt_chars": 11204, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"On-time = carrier_scan_date <= revised_promised_date (or promised_date if no revision)\"\n}\n```\n", "obj": {"answer": 9, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "On-time = carrier_scan_date <= revised_promised_date (or promised_date if no revision)"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-10", "family": "COMP", "rep": 1, "latency_s": 118.57, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 23,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered shipments (customer_confirm_date ≤ 2026-08-31) meeting promised date (revised if applicable).\"\n}\n```\n", "obj": {"answer": 23, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered shipments (customer_confirm_date ≤ 2026-08-31) meeting promised date (revised if applicable)."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-01", "family": "COMP", "rep": 1, "latency_s": 65.04, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers who ordered in past 12 months from 2025-09-01.\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers who ordered in past 12 months from 2025-09-01."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 27.44, "prompt_chars": 11121, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Data snapshot is 2026-09-06; cannot predict future contract status changes\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Data snapshot is 2026-09-06; cannot predict future contract status changes"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-01", "family": "ABST", "rep": 1, "latency_s": 14.38, "prompt_chars": 11125, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No contract closure timestamp available in data to determine churn timing\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No contract closure timestamp available in data to determine churn timing"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-10", "family": "TEMP", "rep": 1, "latency_s": 103.3, "prompt_chars": 15059, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"12 shipments delivered by 2026-02-01 met on-time criteria\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "12 shipments delivered by 2026-02-01 met on-time criteria"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-07", "family": "COMP", "rep": 1, "latency_s": 57.05, "prompt_chars": 12541, "raw": "```json\n{\n  \"answer\": 50,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered = customer_confirm_date populated and ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 50, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date populated and ≤ 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 92.15, "prompt_chars": 16844, "raw": "```json\n{\"answer\": 9, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"9 of 16 C-118 shipments delivered by 2026-07-31 were on time\"}\n```\n", "obj": {"answer": 9, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "9 of 16 C-118 shipments delivered by 2026-07-31 were on time"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 90.11, "prompt_chars": 12561, "raw": "```json\n{\n  \"answer\": 133443,\n  \"definition_id\": \"meridian_net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Q1 invoiced revenue minus Q1 returns only.\"\n}\n```\n", "obj": {"answer": 133443, "definition_id": "meridian_net_revenue", "definition_version": 1, "unanswerable": false, "note": "Q1 invoiced revenue minus Q1 returns only."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 8.21, "prompt_chars": 13950, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No data field tracks shipment damage in transit\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No data field tracks shipment damage in transit"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-11", "family": "XORG", "rep": 1, "latency_s": 30.31, "prompt_chars": 11207, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"C-301 HAL orders with customer_confirm_date ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "C-301 HAL orders with customer_confirm_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-04", "family": "COMP", "rep": 1, "latency_s": 61.17, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 16,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"16 customers with non-cancelled orders in (2025-11-17, 2026-05-15]\"\n}\n```\n", "obj": {"answer": 16, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "16 customers with non-cancelled orders in (2025-11-17, 2026-05-15]"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-03", "family": "COMP", "rep": 1, "latency_s": 84.61, "prompt_chars": 16461, "raw": "```json\n{\"answer\": 17, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Window 180 days back from 2026-06-01: 2025-12-05 to 2026-06-01\"}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window 180 days back from 2026-06-01: 2025-12-05 to 2026-06-01"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 125.56, "prompt_chars": 16514, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Used halden.def.delivered v2 (carrier_scan_date) for delivery; meridian.def.on_time v2 for on-time\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Used halden.def.delivered v2 (carrier_scan_date) for delivery; meridian.def.on_time v2 for on-time"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 49.42, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"active-customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"365-day window ending 2026-03-15; status not cancelled\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "active-customers", "definition_version": 1, "unanswerable": false, "note": "365-day window ending 2026-03-15; status not cancelled"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 45.1, "prompt_chars": 16796, "raw": "```json\n{\"answer\": 9, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Version 1 applied (effective through 2026-01-31)\"}\n```\n", "obj": {"answer": 9, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Version 1 applied (effective through 2026-01-31)"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 46.44, "prompt_chars": 12589, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"delivered_customer_receipt\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Counted HAL shipments for C-118 orders with customer_confirm_date ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "delivered_customer_receipt", "definition_version": 1, "unanswerable": false, "note": "Counted HAL shipments for C-118 orders with customer_confirm_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 83.96, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": \"active_customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"20 active customers with non-cancelled orders in 365-day window ending 2026-02-15\"\n}\n```\n", "obj": {"answer": 20, "definition_id": "active_customers", "definition_version": 1, "unanswerable": false, "note": "20 active customers with non-cancelled orders in 365-day window ending 2026-02-15"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-01", "family": "ABST", "rep": 1, "latency_s": 116.6, "prompt_chars": 16765, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Churn is not defined in catalogue. Cannot determine.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Churn is not defined in catalogue. Cannot determine."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 145.35, "prompt_chars": 15107, "raw": "```json\n{\"answer\": 9, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"13 delivered HAL C-118 shipments; 9 were on-time\"}\n```\n", "obj": {"answer": 9, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "13 delivered HAL C-118 shipments; 9 were on-time"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-04", "family": "ABST", "rep": 1, "latency_s": 10.89, "prompt_chars": 16441, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer satisfaction score field in data\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score field in data"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-06", "family": "XORG", "rep": 1, "latency_s": 53.31, "prompt_chars": 16844, "raw": "```json\n{\n  \"answer\": 8,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"C-118 shipments scanned by 2026-06-30, on-time per revised/promised dates\"\n}\n```\n", "obj": {"answer": 8, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "C-118 shipments scanned by 2026-06-30, on-time per revised/promised dates"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 41.15, "prompt_chars": 16639, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"5 weekdays excluding 2026-01-01 and 2026-01-06 holidays\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "5 weekdays excluding 2026-01-01 and 2026-01-06 holidays"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 9.44, "prompt_chars": 15022, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No damaged-in-transit field in shipments table\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damaged-in-transit field in shipments table"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 62.52, "prompt_chars": 11168, "raw": "```json\n{\n  \"answer\": 141511,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum invoices (2025-10-01 to 2025-12-31) minus returns in period.\"\n}\n```\n", "obj": {"answer": 141511, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum invoices (2025-10-01 to 2025-12-31) minus returns in period."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 111.3, "prompt_chars": 16514, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"9 Halden C-118 shipments delivered and on-time as of cutoff date.\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "9 Halden C-118 shipments delivered and on-time as of cutoff date."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-02", "family": "ABST", "rep": 1, "latency_s": 34.86, "prompt_chars": 12497, "raw": "```json\n{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"Cost of Goods Sold data required for gross margin calculation not provided\"}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Cost of Goods Sold data required for gross margin calculation not provided"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 12.68, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active = contract_status='open'. Static data may not reflect 2026-06-01 state.\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = contract_status='open'. Static data may not reflect 2026-06-01 state."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 8.38, "prompt_chars": 11131, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer satisfaction score column in provided tables\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score column in provided tables"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 94.37, "prompt_chars": 15180, "raw": "```json\n{\n  \"answer\": 18220,\n  \"definition_id\": \"halden.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Two C-118 orders placed Jan-Mar 2026: O004 (7275) + O024 (10945)\"\n}\n```\n", "obj": {"answer": 18220, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Two C-118 orders placed Jan-Mar 2026: O004 (7275) + O024 (10945)"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 23.62, "prompt_chars": 16414, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Gross margin requires COGS data not provided in tables.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Gross margin requires COGS data not provided in tables."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 55.59, "prompt_chars": 13999, "raw": "```json\n{\n  \"answer\": 104670,\n  \"definition_id\": \"net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Invoices 2026-05-01 to 2026-06-30 (freight included from May 1 onwards, less returns)\"\n}\n```\n", "obj": {"answer": 104670, "definition_id": "net_revenue", "definition_version": 2, "unanswerable": false, "note": "Invoices 2026-05-01 to 2026-06-30 (freight included from May 1 onwards, less returns)"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-12", "family": "XORG", "rep": 0, "latency_s": 32.85, "prompt_chars": 14057, "raw": "```json\n{\"answer\": 58155, \"definition_id\": \"meridian_order_value\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"ZEN orders, non-cancelled, 2026-06-01 to 2026-08-31\"}\n```\n", "obj": {"answer": 58155, "definition_id": "meridian_order_value", "definition_version": 1, "unanswerable": false, "note": "ZEN orders, non-cancelled, 2026-06-01 to 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-05", "family": "XORG", "rep": 0, "latency_s": 43.11, "prompt_chars": 12597, "raw": "```json\n{\"answer\": 5, \"definition_id\": \"on_time_performance\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"C-118 HAL shipments delivered by 2026-07-31 vs revised promised dates\"}\n```\n", "obj": {"answer": 5, "definition_id": "on_time_performance", "definition_version": 1, "unanswerable": false, "note": "C-118 HAL shipments delivered by 2026-07-31 vs revised promised dates"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 26.49, "prompt_chars": 16506, "raw": "```json\n{\n  \"answer\": 14,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Halden C-118 orders with carrier_scan_date on/before 2026-06-30\"\n}\n```\n", "obj": {"answer": 14, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "Halden C-118 orders with carrier_scan_date on/before 2026-06-30"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 129.23, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 19,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active definition: ordered in past 12 months ending 2026-01-31\"\n}\n```\n", "obj": {"answer": 19, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active definition: ordered in past 12 months ending 2026-01-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 18.53, "prompt_chars": 16969, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Weekdays excluding Halden holidays 2026-01-01, 2026-01-06.\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Weekdays excluding Halden holidays 2026-01-01, 2026-01-06."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 26.29, "prompt_chars": 11168, "raw": "```json\n{\n  \"answer\": 74641,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Invoiced revenue less returns for period 2026-07-01 to 2026-08-31.\"\n}\n```\n", "obj": {"answer": 74641, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Invoiced revenue less returns for period 2026-07-01 to 2026-08-31."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 82.99, "prompt_chars": 16478, "raw": "```json\n{\n  \"answer\": 105440,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"12 invoiced orders, 1 return; sum=(G-D+F), 111990-6550=105440\"\n}\n```\n", "obj": {"answer": 105440, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "12 invoiced orders, 1 return; sum=(G-D+F), 111990-6550=105440"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-07", "family": "XORG", "rep": 1, "latency_s": 60.21, "prompt_chars": 16917, "raw": "```json\n{\n  \"answer\": 29065,\n  \"definition_id\": \"halden.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-118 supplier HAL orders 2026-05-01 to 2026-07-31, non-cancelled\"\n}\n```\n\n**Calculation details:**\n\nFiltered C-118 orders with order_date between 2026-05-01 and 2026-07-31 (inclusive), status not cancelled:\n- O009 (2026-07-14, invoiced): 4600 - 460 + 350 = 4,490\n- O045 (2026-05-10, invoiced): 3100 - 310 + 150 = 2,940\n- O065 (2026-07-21, invoiced): 16100 - 2415 + 200 = 13,885\n- O072 (2026-05-18, invoiced): 7500 - 0 + 250 = 7,750\n\n**Total: 29,065**\n\nApplied **h", "obj": {"answer": 29065, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "C-118 supplier HAL orders 2026-05-01 to 2026-07-31, non-cancelled"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-10", "family": "XORG", "rep": 1, "latency_s": 39.38, "prompt_chars": 15232, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Counted weekdays excluding Halden holidays between order date and scan date\"\n}\n```\n\n**Calculation:**\n- Order O066 date: 2025-12-28 (Sunday)\n- Shipment S061 carrier scan date: 2026-01-06 (Tuesday)\n- Interval to count: (2025-12-28, 2026-01-06]\n\nBusiness days (Halden definition: weekdays excluding Halden holidays [2026-01-01, 2026-01-06]):\n1. 2025-12-29 (Monday) ✓\n2. 2025-12-30 (Tuesday) ✓\n3. 2025-12-31 (Wednesday) ✓\n4. 2026-01-02 (Friday) ✓\n5. 2026-01-05 (Monday) ", "obj": {"answer": 5, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Counted weekdays excluding Halden holidays between order date and scan date"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 119.0, "prompt_chars": 16844, "raw": "```json\n{\n  \"answer\": 11,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"11 of 19 C-118 shipments delivered by 2026-08-31 on time\"\n}\n```\n", "obj": {"answer": 11, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "11 of 19 C-118 shipments delivered by 2026-08-31 on time"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 95.96, "prompt_chars": 15028, "raw": "```json\n{\n  \"answer\": 3,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Customers active Apr 1 but inactive Jun 30: C007, C013, C021\"\n}\n```\n", "obj": {"answer": 3, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Customers active Apr 1 but inactive Jun 30: C007, C013, C021"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-02", "family": "ABST", "rep": 1, "latency_s": 25.76, "prompt_chars": 16414, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No cost data available; gross margin requires COGS.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No cost data available; gross margin requires COGS."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-06", "family": "XORG", "rep": 1, "latency_s": 136.31, "prompt_chars": 15107, "raw": "```json\n{\n  \"answer\": 8,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Applied halden.def.delivered (carrier_scan_date) for delivery status, meridian.def.on_time v2 for timeliness\"\n}\n```\n", "obj": {"answer": 8, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Applied halden.def.delivered (carrier_scan_date) for delivery status, meridian.def.on_time v2 for timeliness"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-09", "family": "TEMP", "rep": 1, "latency_s": 119.33, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"180-day window from 2025-10-04 through 2026-04-01; 15 unique customers with non-cancelled orders\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "Active customers", "definition_version": 1, "unanswerable": false, "note": "180-day window from 2025-10-04 through 2026-04-01; 15 unique customers with non-cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-03", "family": "TEMP", "rep": 0, "latency_s": 47.49, "prompt_chars": 12544, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Customers with orders in past 12 months from reference date\"\n}\n```\n", "obj": {"answer": 21, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with orders in past 12 months from reference date"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 48.98, "prompt_chars": 16836, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"C-118 shipments with carrier_scan_date ≤ 2026-07-31, per fed.C-118.delivered.\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "C-118 shipments with carrier_scan_date ≤ 2026-07-31, per fed.C-118.delivered."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-07", "family": "XORG", "rep": 1, "latency_s": 76.87, "prompt_chars": 16587, "raw": "```json\n{\n  \"answer\": 29065,\n  \"definition_id\": \"halden.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Four C-118 HAL orders 2026-05-01 to 2026-07-31: O009, O045, O065, O072\"\n}\n```\n", "obj": {"answer": 29065, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Four C-118 HAL orders 2026-05-01 to 2026-07-31: O009, O045, O065, O072"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-05", "family": "COMP", "rep": 1, "latency_s": 32.62, "prompt_chars": 16478, "raw": "```json\n{\n  \"answer\": 104670,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Sum of invoices (gross - discount + freight) minus returns, period 2026-05-01 to 2026-06-30.\"\n}\n```\n", "obj": {"answer": 104670, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Sum of invoices (gross - discount + freight) minus returns, period 2026-05-01 to 2026-06-30."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-10", "family": "XORG", "rep": 1, "latency_s": 23.25, "prompt_chars": 16639, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Order O066 dated 2025-12-28; shipment S061 scanned 2026-01-06. Business days (Mon-Fri, excluding Halden holidays): 2025-12-29, 2025-12-30, 2025-12-31, 2026-01-02, 2026-01-05.\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "Order O066 dated 2025-12-28; shipment S061 scanned 2026-01-06. Business days (Mon-Fri, excluding Halden holidays): 2025-12-29, 2025-12-30, 2025-12-31, 2026-01-02, 2026-01-05."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-02", "family": "ABST", "rep": 1, "latency_s": 18.44, "prompt_chars": 16744, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Gross margin not defined; no cost of goods sold data.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Gross margin not defined; no cost of goods sold data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-12", "family": "XORG", "rep": 1, "latency_s": 35.68, "prompt_chars": 11226, "raw": "```json\n{\n  \"answer\": 62700,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of gross_amount for non-cancelled ZEN orders, 2026-06-01 to 2026-08-31\"\n}\n```\n", "obj": {"answer": 62700, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of gross_amount for non-cancelled ZEN orders, 2026-06-01 to 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-08", "family": "XORG", "rep": 1, "latency_s": 91.26, "prompt_chars": 11277, "raw": "```json\n{\n  \"answer\": 21624,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"HAL C-118 non-cancelled orders placed 2026-01-01 to 2026-03-31: O004 (8628) + O024 (12996)\"\n}\n```\n", "obj": {"answer": 21624, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "HAL C-118 non-cancelled orders placed 2026-01-01 to 2026-03-31: O004 (8628) + O024 (12996)"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 92.37, "prompt_chars": 13999, "raw": "```json\n{\n  \"answer\": 109343,\n  \"definition_id\": \"net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"13 invoices (gross-discount) minus 1 return; freight excluded per v1.\"\n}\n```\n", "obj": {"answer": 109343, "definition_id": "net_revenue", "definition_version": 1, "unanswerable": false, "note": "13 invoices (gross-discount) minus 1 return; freight excluded per v1."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-01", "family": "ABST", "rep": 1, "latency_s": 203.37, "prompt_chars": 16435, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Compared active customers on March 31 vs June 30, 2026 using versions 1 and 2.\"\n}\n```\n", "obj": {"answer": 6, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Compared active customers on March 31 vs June 30, 2026 using versions 1 and 2."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-06", "family": "XORG", "rep": 1, "latency_s": 128.91, "prompt_chars": 14035, "raw": "```json\n{\n  \"answer\": 8,\n  \"definition_id\": \"on_time_delivery\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Applied Halden delivered rule (carrier_scan≤2026-06-30), Meridian on-time v2\"\n}\n```\n", "obj": {"answer": 8, "definition_id": "on_time_delivery", "definition_version": 2, "unanswerable": false, "note": "Applied Halden delivered rule (carrier_scan≤2026-06-30), Meridian on-time v2"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 68.54, "prompt_chars": 16796, "raw": "```json\n{\"answer\": 7, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"19 shipments delivered by 2025-12-31; 7 met promised_date\"}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "19 shipments delivered by 2025-12-31; 7 met promised_date"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-01", "family": "ABST", "rep": 1, "latency_s": 70.27, "prompt_chars": 13956, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": \"Active customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customers active at Q1 end but inactive at Q2 end: C013, C016, C018, C019, C021, C022.\"\n}\n```\n", "obj": {"answer": 6, "definition_id": "Active customers", "definition_version": 1, "unanswerable": false, "note": "Customers active at Q1 end but inactive at Q2 end: C013, C016, C018, C019, C021, C022."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-04", "family": "ABST", "rep": 1, "latency_s": 9.52, "prompt_chars": 16771, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer satisfaction score data provided in tables\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score data provided in tables"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-03", "family": "COMP", "rep": 1, "latency_s": 13.89, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active = customers with open contract_status as of 2026-06-01\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = customers with open contract_status as of 2026-06-01"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-03", "family": "TEMP", "rep": 1, "latency_s": 43.48, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Active: non-cancelled orders within 365 days prior, 2025-03-16 to 2026-03-15\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Active: non-cancelled orders within 365 days prior, 2025-03-16 to 2026-03-15"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-09", "family": "COMP", "rep": 0, "latency_s": 86.63, "prompt_chars": 12541, "raw": "```json\n{\"answer\": 37, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date populated and on/before 2026-05-31\"}\n```\n", "obj": {"answer": 37, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date populated and on/before 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-07", "family": "TEMP", "rep": 0, "latency_s": 38.17, "prompt_chars": 16808, "raw": "```json\n{\n  \"answer\": 109343,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Sum of (gross_amount - discount_amount) for 13 invoices dated 2026-01-01 to 2026-03-31, minus 1 return.\"\n}\n```\n", "obj": {"answer": 109343, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "Sum of (gross_amount - discount_amount) for 13 invoices dated 2026-01-01 to 2026-03-31, minus 1 return."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-05", "family": "COMP", "rep": 1, "latency_s": 57.17, "prompt_chars": 15071, "raw": "```json\n{\"answer\": 104670, \"definition_id\": \"meridian.def.net_revenue\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Invoices 2026-05-01 to 2026-06-30: 111220; Returns: 6550\"}\n```\n", "obj": {"answer": 104670, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Invoices 2026-05-01 to 2026-06-30: 111220; Returns: 6550"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 33.35, "prompt_chars": 12589, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"delivery_by_customer_confirmation\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Counted C-118 HAL shipments with customer_confirm_date ≤ 2026-06-30\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "delivery_by_customer_confirmation", "definition_version": 1, "unanswerable": false, "note": "Counted C-118 HAL shipments with customer_confirm_date ≤ 2026-06-30"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 89.11, "prompt_chars": 12561, "raw": "```json\n{\n  \"answer\": 61003,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Gross sales minus discounts minus returns, excluding freight and tax pass-throughs.\"\n}\n```\n", "obj": {"answer": 61003, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Gross sales minus discounts minus returns, excluding freight and tax pass-throughs."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-09", "family": "TEMP", "rep": 1, "latency_s": 67.69, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Version 2 effective 2026-04-01: 180-day window from 2025-10-05.\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Version 2 effective 2026-04-01: 180-day window from 2025-10-05."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 54.54, "prompt_chars": 11329, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Business days (Mon–Fri excluding 2026-01-01 holiday) from 2025-12-31 to 2026-01-14\"\n}\n```\n", "obj": {"answer": 10, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Business days (Mon–Fri excluding 2026-01-01 holiday) from 2025-12-31 to 2026-01-14"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-09", "family": "TEMP", "rep": 0, "latency_s": 43.55, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 13,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Window 180 days back from 2026-04-01: (2025-10-04, 2026-04-01]\"\n}\n```\n", "obj": {"answer": 13, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window 180 days back from 2026-04-01: (2025-10-04, 2026-04-01]"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-04", "family": "XORG", "rep": 1, "latency_s": 98.7, "prompt_chars": 11204, "raw": "Looking at the data, I need to:\n1. Identify all orders with contract_id = \"C-118\" and supplier_id = \"HAL\" (Halden Logistics)\n2. Find their corresponding shipments with carrier_scan_dates (delivered)\n3. Check if delivery was on or before the promised_date (or revised_promised_date if available)\n4. Count those delivered as of 2026-08-31\n\n**C-118 orders with HAL supplier:**\nO004, O006, O007, O013, O017, O021, O024, O039, O045, O047, O048, O051 (cancelled), O052, O062, O065, O066, O072\n\n**Shipment analysis (carrier_scan_date vs promised delivery date):**\n\n| Order | Carrier Scan | Promised | Revise", "obj": {"answer": 10, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "On-time = carrier_scan_date <= promised_date or revised_promised_date"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 85.35, "prompt_chars": 11125, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No churn date timestamp available; only current contract_status.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No churn date timestamp available; only current contract_status."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-05", "family": "ABST", "rep": 1, "latency_s": 168.59, "prompt_chars": 16813, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Counted deliveries (customer_confirm_date present) late by >2 business days using meridian.def.business_day; excludes 2025 dates (holiday data unavailable).\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "Counted deliveries (customer_confirm_date present) late by >2 business days using meridian.def.business_day; excludes 2025 dates (holiday data unavailable)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 63.34, "prompt_chars": 13999, "raw": "```json\n{\n  \"answer\": 62903,\n  \"definition_id\": \"net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Freight included per v2 rules (from 1 May 2026). One return deducted.\"\n}\n```\n", "obj": {"answer": 62903, "definition_id": "net_revenue", "definition_version": 2, "unanswerable": false, "note": "Freight included per v2 rules (from 1 May 2026). One return deducted."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 51.71, "prompt_chars": 12670, "raw": "```json\n{\n  \"answer\": 21624,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Sum of invoiced amounts for HAL-supplied C-118 orders placed Jan–Mar 2026.\"\n}\n```\n", "obj": {"answer": 21624, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Sum of invoiced amounts for HAL-supplied C-118 orders placed Jan–Mar 2026."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-06", "family": "ABST", "rep": 1, "latency_s": 72.56, "prompt_chars": 12514, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": \"past_year\",\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Missing order data for Q4 2026; cannot count past-year active customers\"\n}\n```\n", "obj": {"answer": null, "definition_id": "past_year", "definition_version": null, "unanswerable": true, "note": "Missing order data for Q4 2026; cannot count past-year active customers"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-10", "family": "XORG", "rep": 1, "latency_s": 21.63, "prompt_chars": 16969, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"5 business days between 2025-12-28 and 2026-01-06\"\n}\n```\n\n**Calculation:**\n\nOrder O066: order_date = 2025-12-28, supplier = HAL, contract = C-118\nShipment S061: carrier_scan_date = 2026-01-06\n\nBusiness days in the interval (2025-12-28, 2026-01-06] under halden.def.business_day v1:\n- 2025-12-29 (Monday) ✓\n- 2025-12-30 (Tuesday) ✓\n- 2025-12-31 (Wednesday) ✓\n- 2026-01-01 (Thursday) ✗ Halden holiday\n- 2026-01-02 (Friday) ✓\n- 2026-01-03 (Saturday) ✗ Weekend\n- 2026-01", "obj": {"answer": 5, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "5 business days between 2025-12-28 and 2026-01-06"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-03", "family": "XORG", "rep": 1, "latency_s": 120.32, "prompt_chars": 14027, "raw": "```json\n{\n  \"answer\": 14,\n  \"definition_id\": \"delivered-shipments\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden rule: carrier_scan_date present and on/before 2026-07-31\"\n}\n```\n", "obj": {"answer": 14, "definition_id": "delivered-shipments", "definition_version": 1, "unanswerable": false, "note": "Halden rule: carrier_scan_date present and on/before 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 65.91, "prompt_chars": 13979, "raw": "```json\n{\n  \"answer\": 50,\n  \"definition_id\": \"Delivered shipments\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Meridian rule: customer_confirm_date present and ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 50, "definition_id": "Delivered shipments", "definition_version": 1, "unanswerable": false, "note": "Meridian rule: customer_confirm_date present and ≤ 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-01", "family": "TEMP", "rep": 1, "latency_s": 44.02, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"21 customers with non-cancelled orders in (2025-02-28, 2026-02-28].\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "21 customers with non-cancelled orders in (2025-02-28, 2026-02-28]."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-07", "family": "TEMP", "rep": 1, "latency_s": 28.71, "prompt_chars": 11168, "raw": "```json\n{\n  \"answer\": 133443,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Q1 2026 invoices minus returns: 137,020 - 3,577\"\n}\n```\n", "obj": {"answer": 133443, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Q1 2026 invoices minus returns: 137,020 - 3,577"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-01", "family": "XORG", "rep": 0, "latency_s": 104.97, "prompt_chars": 11196, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"C-118 shipments with customer_confirm_date ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 15, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "C-118 shipments with customer_confirm_date ≤ 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-03", "family": "XORG", "rep": 1, "latency_s": 43.17, "prompt_chars": 15099, "raw": "```json\n{\n  \"answer\": 15,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Counted HAL shipments with carrier_scan_date on or before 2026-07-31 for C-118 contract orders.\"\n}\n```\n", "obj": {"answer": 15, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "Counted HAL shipments with carrier_scan_date on or before 2026-07-31 for C-118 contract orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-06", "family": "COMP", "rep": 0, "latency_s": 41.41, "prompt_chars": 16808, "raw": "```json\n{\n  \"answer\": 62903,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"7 invoices in period, 1 return. Formula: (gross - discount + freight) minus returns.\"\n}\n```\n", "obj": {"answer": 62903, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "7 invoices in period, 1 return. Formula: (gross - discount + freight) minus returns."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-02", "family": "TEMP", "rep": 0, "latency_s": 12.53, "prompt_chars": 11151, "raw": "```json\n{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active customers = open contract status as of 2026-01-31\"}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active customers = open contract status as of 2026-01-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 53.29, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"on-time-delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"10 shipments were on-time per pre-Feb-2026 Meridian rule.\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "on-time-delivery", "definition_version": 1, "unanswerable": false, "note": "10 shipments were on-time per pre-Feb-2026 Meridian rule."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-02", "family": "TEMP", "rep": 1, "latency_s": 24.87, "prompt_chars": 12544, "raw": "```json\n{\"answer\": 19, \"definition_id\": \"past_year_orders\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Customers with orders placed 2025-01-31 to 2026-01-31\"}\n```\n", "obj": {"answer": 19, "definition_id": "past_year_orders", "definition_version": 1, "unanswerable": false, "note": "Customers with orders placed 2025-01-31 to 2026-01-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 15.8, "prompt_chars": 11151, "raw": "```json\n{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Customers with open contract_status as of 2026-08-31\"}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with open contract_status as of 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 48.07, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Applied v2 (180-day window from 2026-03-04). Counted unique customers with non-cancelled orders.\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Applied v2 (180-day window from 2026-03-04). Counted unique customers with non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-11", "family": "COMP", "rep": 1, "latency_s": 113.48, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"45 shipments delivered by 2026-07-31; 21 met promised or revised promised date\"\n}\n```\n", "obj": {"answer": 21, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "45 shipments delivered by 2026-07-31; 21 met promised or revised promised date"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-03", "family": "TEMP", "rep": 1, "latency_s": 45.17, "prompt_chars": 12544, "raw": "```json\n{\"answer\": 21, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Customers with orders 2025-03-15 to 2026-03-15\"}\n```\n", "obj": {"answer": 21, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Customers with orders 2025-03-15 to 2026-03-15"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-06", "family": "XORG", "rep": 0, "latency_s": 96.29, "prompt_chars": 14035, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"On-time_delivery\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Halden delivery rule from 2026-02-01; 7 of 13 delivered shipments on-time\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "On-time_delivery", "definition_version": 2, "unanswerable": false, "note": "Halden delivery rule from 2026-02-01; 7 of 13 delivered shipments on-time"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-05", "family": "COMP", "rep": 0, "latency_s": 33.15, "prompt_chars": 11168, "raw": "```json\n{\n  \"answer\": 125200,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Invoiced revenue minus returns, period 2026-05-01 to 2026-06-30\"\n}\n```\n", "obj": {"answer": 125200, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Invoiced revenue minus returns, period 2026-05-01 to 2026-06-30"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 9.59, "prompt_chars": 16771, "raw": "```json\n{\"answer\": null, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": true, \"note\": \"No customer satisfaction score field in data\"}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score field in data"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 8.63, "prompt_chars": 16429, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No damage information recorded in shipments table.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damage information recorded in shipments table."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-06", "family": "TEMP", "rep": 0, "latency_s": 47.39, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered = customer_confirm_date ≤ 2025-12-31; on-time = confirmed ≤ revised_promised_date or promised_date\"\n}\n```\n", "obj": {"answer": 9, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date ≤ 2025-12-31; on-time = confirmed ≤ revised_promised_date or promised_date"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 54.43, "prompt_chars": 16461, "raw": "```json\n{\"answer\": 20, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"365-day window: (2025-02-15, 2026-02-15], non-cancelled orders\"}\n```\n", "obj": {"answer": 20, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "365-day window: (2025-02-15, 2026-02-15], non-cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-07", "family": "TEMP", "rep": 1, "latency_s": 48.49, "prompt_chars": 13999, "raw": "```json\n{\n  \"answer\": 109343,\n  \"definition_id\": \"Net revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Freight excluded; freight period not yet reached.\"\n}\n```\n", "obj": {"answer": 109343, "definition_id": "Net revenue", "definition_version": 1, "unanswerable": false, "note": "Freight excluded; freight period not yet reached."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-10", "family": "TEMP", "rep": 1, "latency_s": 133.76, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"on-time-delivery\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"12 shipments delivered by 2026-02-01 were on-time using Meridian v2 rule.\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "on-time-delivery", "definition_version": 2, "unanswerable": false, "note": "12 shipments delivered by 2026-02-01 were on-time using Meridian v2 rule."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-06", "family": "XORG", "rep": 1, "latency_s": 48.51, "prompt_chars": 11204, "raw": "```json\n{\n  \"answer\": 8,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"C-118 shipments with carrier_scan_date <= 2026-06-30 compared to promised/revised date\"\n}\n```\n", "obj": {"answer": 8, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "C-118 shipments with carrier_scan_date <= 2026-06-30 compared to promised/revised date"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-12", "family": "XORG", "rep": 1, "latency_s": 27.12, "prompt_chars": 12619, "raw": "```json\n{\"answer\": 62700, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Seven ZEN orders June-August 2026, non-cancelled; gross_amount summed.\"}\n```\n", "obj": {"answer": 62700, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Seven ZEN orders June-August 2026, non-cancelled; gross_amount summed."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-04", "family": "TEMP", "rep": 1, "latency_s": 36.1, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Window: (2025-02-15, 2026-02-15]. 20 unique customers have non-cancelled orders.\"\n}\n```\n", "obj": {"answer": 20, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Window: (2025-02-15, 2026-02-15]. 20 unique customers have non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 42.6, "prompt_chars": 15099, "raw": "```json\n{\n  \"answer\": 13,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"15 C-118 shipments exist; 13 have carrier_scan_date ≤ 2026-07-31\"\n}\n```\n", "obj": {"answer": 13, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "15 C-118 shipments exist; 13 have carrier_scan_date ≤ 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "XORG-11", "family": "XORG", "rep": 1, "latency_s": 29.06, "prompt_chars": 16847, "raw": "```json\n{\n  \"answer\": 10,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C-301 shipments with customer_confirm_date present and on/before 2026-08-31\"\n}\n```\n", "obj": {"answer": 10, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "C-301 shipments with customer_confirm_date present and on/before 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-01", "family": "XORG", "rep": 1, "latency_s": 89.29, "prompt_chars": 14027, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"Delivered shipments\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden rule: carrier scan date present and ≤ 2026-08-31\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "Delivered shipments", "definition_version": 1, "unanswerable": false, "note": "Halden rule: carrier scan date present and ≤ 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-04", "family": "XORG", "rep": 0, "latency_s": 45.78, "prompt_chars": 12597, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered = has customer_confirm_date. On-time = delivered <= promised or revised date.\"\n}\n```\n", "obj": {"answer": 6, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = has customer_confirm_date. On-time = delivered <= promised or revised date."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-06", "family": "TEMP", "rep": 1, "latency_s": 101.29, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"On-time = customer_confirm_date ≤ promised/revised_promised_date; delivered by 2025-12-31 = customer_confirm_date ≤ 2025-12-31 and NOT null\"\n}\n```\n\n**Calculation summary:**\n\nShipments delivered by 2025-12-31 (customer_confirm_date populated and ≤ 2025-12-31): 21 total\n\nDelivered on time (customer_confirm_date ≤ promised_date, or revised_promised_date if present):\n1. S010 (O011): 2025-06-20 ≤ 2025-06-22 ✓\n2. S012 (O013): 2025-08-17 ≤ 2025-08-21 ✓\n3. S014 (O015): 2025-12-30 ≤ 2026-", "obj": {"answer": 9, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "On-time = customer_confirm_date ≤ promised/revised_promised_date; delivered by 2025-12-31 = customer_confirm_date ≤ 2025-12-31 and NOT null"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-01", "family": "COMP", "rep": 1, "latency_s": 65.52, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"Active customers - Meridian reporting\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"180-day window from 2026-03-05 to 2026-08-31; excluded cancelled orders\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "Active customers - Meridian reporting", "definition_version": 1, "unanswerable": false, "note": "180-day window from 2026-03-05 to 2026-08-31; excluded cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 41.77, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date ≤ promised_date (or revised_promised_date if present) as of 2026-02-01\"\n}\n```\n", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with customer_confirm_date ≤ promised_date (or revised_promised_date if present) as of 2026-02-01"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 149.51, "prompt_chars": 12518, "raw": "```json\n{\n  \"answer\": 1,\n  \"definition_id\": \"active_base_ordering_past_year\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"C002 fell out of active base (no orders in past 12 months from Q2 end)\"\n}\n```\n", "obj": {"answer": 1, "definition_id": "active_base_ordering_past_year", "definition_version": 1, "unanswerable": false, "note": "C002 fell out of active base (no orders in past 12 months from Q2 end)"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 18.05, "prompt_chars": 12722, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Business days exclude weekends and Jan 1 holiday\"\n}\n```\n", "obj": {"answer": 6, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Business days exclude weekends and Jan 1 holiday"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 68.82, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"12 of 25 delivered shipments met on-time criteria.\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "12 of 25 delivered shipments met on-time criteria."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 8.12, "prompt_chars": 13962, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer satisfaction score field in data\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score field in data"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 90.37, "prompt_chars": 16435, "raw": "```json\n{\n  \"answer\": 2,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Customers C013 and C021 were active at Q2 start (2026-04-01) but inactive by Q2 end (2026-06-30).\"\n}\n```\n", "obj": {"answer": 2, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Customers C013 and C021 were active at Q2 start (2026-04-01) but inactive by Q2 end (2026-06-30)."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 34.21, "prompt_chars": 14027, "raw": "```json\n{\n  \"answer\": 16,\n  \"definition_id\": \"Delivered shipments\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Halden rule: carrier_scan_date present and on/before 2026-07-31\"\n}\n```\n", "obj": {"answer": 16, "definition_id": "Delivered shipments", "definition_version": 1, "unanswerable": false, "note": "Halden rule: carrier_scan_date present and on/before 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 13.22, "prompt_chars": 11151, "raw": "```json\n{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active = contract_status='open' as of 2026-07-15\"}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = contract_status='open' as of 2026-07-15"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-07", "family": "TEMP", "rep": 1, "latency_s": 121.31, "prompt_chars": 12561, "raw": "```json\n{\n  \"answer\": 133443,\n  \"definition_id\": \"meridian-revenue-q1-2026\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Invoiced revenue Q1 2026 less returns recognized in period.\"\n}\n```\n\n**Calculation:**\n\n**Invoices dated 2026-01-01 to 2026-03-31:**\n- I003 (O004, 2026-02-03): 8,628\n- I005 (O006, 2026-01-18): 9,501\n- I018 (O023, 2026-03-01): 17,178\n- I019 (O024, 2026-03-15): 12,996\n- I023 (O028, 2026-01-21): 18,219\n- I025 (O030, 2026-01-04): 8,775\n- I040 (O053, 2026-02-16): 4,105\n- I041 (O054, 2026-01-30): 15,736\n- I042 (O056, 2026-03-31): 5,755\n- I043 (O057, 2026-02-28): 5", "obj": {"answer": 133443, "definition_id": "meridian-revenue-q1-2026", "definition_version": 1, "unanswerable": false, "note": "Invoiced revenue Q1 2026 less returns recognized in period."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 69.47, "prompt_chars": 15059, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"12 of 25 delivered shipments met on-time criteria.\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "12 of 25 delivered shipments met on-time criteria."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-10", "family": "TEMP", "rep": 1, "latency_s": 41.85, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with confirmed delivery date ≤ revised promised date or promised date\"\n}\n```\n", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with confirmed delivery date ≤ revised promised date or promised date"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-08", "family": "COMP", "rep": 1, "latency_s": 53.18, "prompt_chars": 16458, "raw": "```json\n{\n  \"answer\": 43,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date present and on/before 2026-06-30\"\n}\n```\n", "obj": {"answer": 43, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present and on/before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-08", "family": "COMP", "rep": 1, "latency_s": 45.8, "prompt_chars": 15051, "raw": "```json\n{\n  \"answer\": 43,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date on or before 2026-06-30\"\n}\n```\n", "obj": {"answer": 43, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date on or before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-02", "family": "ABST", "rep": 0, "latency_s": 42.39, "prompt_chars": 12497, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"COGS data required to calculate gross margin; dataset lacks cost information.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "COGS data required to calculate gross margin; dataset lacks cost information."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-04", "family": "COMP", "rep": 1, "latency_s": 40.62, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 16,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Window: 2025-11-17 to 2026-05-15, status not cancelled\"\n}\n```\n", "obj": {"answer": 16, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window: 2025-11-17 to 2026-05-15, status not cancelled"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "XORG-11", "family": "XORG", "rep": 1, "latency_s": 14.91, "prompt_chars": 14038, "raw": "```json\n{\n  \"answer\": 11,\n  \"definition_id\": \"Delivered shipments\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Meridian rule: customer_confirm_date present and on/before 2026-08-31\"\n}\n```\n", "obj": {"answer": 11, "definition_id": "Delivered shipments", "definition_version": 1, "unanswerable": false, "note": "Meridian rule: customer_confirm_date present and on/before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-08", "family": "TEMP", "rep": 1, "latency_s": 39.03, "prompt_chars": 12561, "raw": "```json\n{\n  \"answer\": 141511,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Invoiced sales less returns in period\"\n}\n```\n", "obj": {"answer": 141511, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Invoiced sales less returns in period"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-01", "family": "TEMP", "rep": 1, "latency_s": 46.39, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"21 unique customers with non-cancelled orders in (2025-02-28, 2026-02-28]\"\n}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "21 unique customers with non-cancelled orders in (2025-02-28, 2026-02-28]"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 53.25, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered shipments with customer_confirm_date ≤ revised_promised_date or promised_date.\"\n}\n```\n", "obj": {"answer": 21, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered shipments with customer_confirm_date ≤ revised_promised_date or promised_date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "COMP-04", "family": "COMP", "rep": 1, "latency_s": 57.95, "prompt_chars": 13982, "raw": "```json\n{\n  \"answer\": 16,\n  \"definition_id\": \"active_customers\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day window from 2025-11-16 to 2026-05-15; 16 unique customers with non-cancelled orders.\"\n}\n```\n", "obj": {"answer": 16, "definition_id": "active_customers", "definition_version": 2, "unanswerable": false, "note": "180-day window from 2025-11-16 to 2026-05-15; 16 unique customers with non-cancelled orders."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-08", "family": "COMP", "rep": 1, "latency_s": 44.32, "prompt_chars": 11148, "raw": "```json\n{\n  \"answer\": 43,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with non-null customer_confirm_date on/before 2026-06-30\"\n}\n```\n", "obj": {"answer": 43, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with non-null customer_confirm_date on/before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-12", "family": "XORG", "rep": 1, "latency_s": 37.81, "prompt_chars": 15129, "raw": "```json\n{\n  \"answer\": 58155,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Sum of (gross_amount - discount_amount) for non-cancelled ZEN orders placed 2026-06-01 to 2026-08-31\"\n}\n```\n", "obj": {"answer": 58155, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum of (gross_amount - discount_amount) for non-cancelled ZEN orders placed 2026-06-01 to 2026-08-31"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-10", "family": "XORG", "rep": 0, "latency_s": 78.47, "prompt_chars": 15232, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"5 business days between order and carrier scan dates\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "5 business days between order and carrier scan dates"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 87.87, "prompt_chars": 16431, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Five customers with non-cancelled orders in window (2026-07-05, 2027-01-01].\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Five customers with non-cancelled orders in window (2026-07-05, 2027-01-01]."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 41.41, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Counted customers with contract_status='open' as of 2026-02-15\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Counted customers with contract_status='open' as of 2026-02-15"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "TEMP-05", "family": "TEMP", "rep": 0, "latency_s": 44.44, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered = customer_confirm_date ≤ 2026-01-31; On-time = customer_confirm_date ≤ revised_promised_date (if exists) else promised_date\"\n}\n```\n", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date ≤ 2026-01-31; On-time = customer_confirm_date ≤ revised_promised_date (if exists) else promised_date"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "TEMP-10", "family": "TEMP", "rep": 1, "latency_s": 76.74, "prompt_chars": 12549, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"meridian_ontime_delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"12 of 25 delivered shipments met promised/revised delivery dates\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "meridian_ontime_delivery", "definition_version": 1, "unanswerable": false, "note": "12 of 25 delivered shipments met promised/revised delivery dates"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-10", "family": "COMP", "rep": 0, "latency_s": 55.94, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 23,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Delivered shipments: actual customer_confirm_date ≤ promised/revised date.\"\n}\n```\n", "obj": {"answer": 23, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered shipments: actual customer_confirm_date ≤ promised/revised date."}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-04", "family": "COMP", "rep": 1, "latency_s": 13.3, "prompt_chars": 11151, "raw": "```json\n{\n  \"answer\": 20,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Active = open contract status as of date\"\n}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = open contract status as of date"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-10", "family": "COMP", "rep": 1, "latency_s": 192.64, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 23,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"23 delivered shipments met on-time criteria as of 2026-08-31.\"\n}\n```\n", "obj": {"answer": 23, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "23 delivered shipments met on-time criteria as of 2026-08-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-01", "family": "TEMP", "rep": 1, "latency_s": 36.99, "prompt_chars": 16791, "raw": "```json\n{\"answer\": 21, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Customers with non-cancelled orders in (2025-02-28, 2026-02-28]\"}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Customers with non-cancelled orders in (2025-02-28, 2026-02-28]"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-04", "family": "ABST", "rep": 1, "latency_s": 8.38, "prompt_chars": 15034, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer satisfaction score data in provided tables.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score data in provided tables."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-07", "family": "COMP", "rep": 0, "latency_s": 23.35, "prompt_chars": 16458, "raw": "```json\n{\n  \"answer\": 51,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date present and on/before 2026-08-31\"\n}\n```\n", "obj": {"answer": 51, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present and on/before 2026-08-31"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 96.57, "prompt_chars": 16765, "raw": "```json\n{\"answer\": 5, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Customers active 2026-03-31 but inactive 2026-06-30: C016, C018, C019, C021, C022\"}\n```\n", "obj": {"answer": 5, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "Customers active 2026-03-31 but inactive 2026-06-30: C016, C018, C019, C021, C022"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 9.17, "prompt_chars": 16759, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Shipments table lacks damage tracking data or status field\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Shipments table lacks damage tracking data or status field"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-04", "family": "TEMP", "rep": 0, "latency_s": 64.3, "prompt_chars": 15054, "raw": "```json\n{\"answer\": 20, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"20 active customers in 365-day window ending 2026-02-15\"}\n```\n", "obj": {"answer": 20, "definition_id": "meridian.def.active_customer", "definition_version": 1, "unanswerable": false, "note": "20 active customers in 365-day window ending 2026-02-15"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-06", "family": "ABST", "rep": 1, "latency_s": 54.49, "prompt_chars": 15024, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Window (2026-07-05, 2027-01-01]: customers C001, C004, C011, C013, C022\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window (2026-07-05, 2027-01-01]: customers C001, C004, C011, C013, C022"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-01", "family": "COMP", "rep": 0, "latency_s": 62.43, "prompt_chars": 12544, "raw": "```json\n{\"answer\": 20, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Active = ordered in past 12 months (≥2025-08-31)\"}\n```\n", "obj": {"answer": 20, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Active = ordered in past 12 months (≥2025-08-31)"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 32.26, "prompt_chars": 16458, "raw": "```json\n{\n  \"answer\": 43,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date present and on/before 2026-06-30.\"\n}\n```\n", "obj": {"answer": 43, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date present and on/before 2026-06-30."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "TEMP-10", "family": "TEMP", "rep": 0, "latency_s": 37.39, "prompt_chars": 13987, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"on-time-delivery\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Applied post-2026-02-01 rule using revised promised dates where populated\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "on-time-delivery", "definition_version": 2, "unanswerable": false, "note": "Applied post-2026-02-01 rule using revised promised dates where populated"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-05", "family": "ABST", "rep": 1, "latency_s": 190.31, "prompt_chars": 14004, "raw": "```json\n{\n  \"answer\": 18,\n  \"definition_id\": \"on-time-delivery\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Counted delivered orders where delivery date exceeded promised/revised date by more than 2 business days using Meridian calendar per the COO rule effective from 2026-02-01\"\n}\n```\n", "obj": {"answer": 18, "definition_id": "on-time-delivery", "definition_version": 2, "unanswerable": false, "note": "Counted delivered orders where delivery date exceeded promised/revised date by more than 2 business days using Meridian calendar per the COO rule effective from 2026-02-01"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-11", "family": "XORG", "rep": 0, "latency_s": 37.09, "prompt_chars": 15110, "raw": "```json\n{\n  \"answer\": 11,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customer_confirm_date present and on/before 2026-08-31 for C-301 HAL shipments\"\n}\n```\n", "obj": {"answer": 11, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Customer_confirm_date present and on/before 2026-08-31 for C-301 HAL shipments"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 19.65, "prompt_chars": 16788, "raw": "```json\n{\n  \"answer\": 43,\n  \"definition_id\": \"meridian.def.delivered\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Counted shipments with customer_confirm_date present and on/before 2026-06-30\"\n}\n```\n", "obj": {"answer": 43, "definition_id": "meridian.def.delivered", "definition_version": 1, "unanswerable": false, "note": "Counted shipments with customer_confirm_date present and on/before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "ABST-03", "family": "ABST", "rep": 0, "latency_s": 7.9, "prompt_chars": 11119, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Shipments table lacks damage tracking information.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Shipments table lacks damage tracking information."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "XORG-08", "family": "XORG", "rep": 0, "latency_s": 65.27, "prompt_chars": 16587, "raw": "```json\n{\n  \"answer\": 18220,\n  \"definition_id\": \"halden.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Two non-cancelled C-118 HAL orders placed in period: O004 (7275) + O024 (10945)\"\n}\n```\n", "obj": {"answer": 18220, "definition_id": "halden.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Two non-cancelled C-118 HAL orders placed in period: O004 (7275) + O024 (10945)"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "ABST-04", "family": "ABST", "rep": 1, "latency_s": 9.38, "prompt_chars": 12524, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Customer satisfaction scores not present in data.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Customer satisfaction scores not present in data."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "COMP-11", "family": "COMP", "rep": 1, "latency_s": 67.75, "prompt_chars": 11156, "raw": "```json\n{\n  \"answer\": 21,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with customer_confirm_date ≤ 2026-07-31, delivered on or before promised/revised date\"\n}\n```\n", "obj": {"answer": 21, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Shipments with customer_confirm_date ≤ 2026-07-31, delivered on or before promised/revised date"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "TEMP-06", "family": "TEMP", "rep": 1, "latency_s": 54.22, "prompt_chars": 16466, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments delivered with customer_confirm_date by 2025-12-31, on or before promised_date.\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "Shipments delivered with customer_confirm_date by 2025-12-31, on or before promised_date."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "ABST-04", "family": "ABST", "rep": 0, "latency_s": 8.17, "prompt_chars": 15034, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No customer satisfaction score data in provided tables\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No customer satisfaction score data in provided tables"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-08", "family": "COMP", "rep": 1, "latency_s": 41.89, "prompt_chars": 12541, "raw": "```json\n{\"answer\": 43, \"definition_id\": \"delivery_definition_context\", \"definition_version\": 1, \"unanswerable\": false, \"note\": \"Shipments with customer_confirm_date on/before 2026-06-30\"}\n```\n", "obj": {"answer": 43, "definition_id": "delivery_definition_context", "definition_version": 1, "unanswerable": false, "note": "Shipments with customer_confirm_date on/before 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "TEMP-06", "family": "TEMP", "rep": 1, "latency_s": 58.66, "prompt_chars": 15059, "raw": "```json\n{\n  \"answer\": 7,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"7 of 20 delivered shipments met promised_date\"\n}\n```\n", "obj": {"answer": 7, "definition_id": "meridian.def.on_time", "definition_version": 1, "unanswerable": false, "note": "7 of 20 delivered shipments met promised_date"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 64.36, "prompt_chars": 11196, "raw": "```json\n{\n  \"answer\": 14,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"C-118 + Halden Logistics shipments with carrier scan date ≤ 2026-06-30\"\n}\n```\n", "obj": {"answer": 14, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "C-118 + Halden Logistics shipments with carrier scan date ≤ 2026-06-30"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-03", "family": "COMP", "rep": 0, "latency_s": 96.51, "prompt_chars": 16461, "raw": "```json\n{\"answer\": 17, \"definition_id\": \"meridian.def.active_customer\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"Window (2025-12-03, 2026-06-01]; non-cancelled orders only\"}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window (2025-12-03, 2026-06-01]; non-cancelled orders only"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-04", "family": "COMP", "rep": 0, "latency_s": 43.37, "prompt_chars": 16791, "raw": "```json\n{\n  \"answer\": 16,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Window: 2025-11-16 to 2026-05-15; 16 unique customers with non-cancelled orders\"\n}\n```\n", "obj": {"answer": 16, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Window: 2025-11-16 to 2026-05-15; 16 unique customers with non-cancelled orders"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "schema", "task": "XORG-03", "family": "XORG", "rep": 0, "latency_s": 69.77, "prompt_chars": 11196, "raw": "Looking at the data, I need to:\n1. Find all orders with contract_id = C-118 and supplier_id = HAL\n2. Identify corresponding shipments\n3. Count those with a non-null customer_confirm_date on or before 2026-07-31\n\n**C-118 orders with HAL supplier:**\nO004, O006, O007, O009, O013, O017, O021, O022, O023, O024, O039, O041, O045, O047, O048, O051, O052, O062, O065, O066, O072\n\n**Checking shipments with customer_confirm_date ≤ 2026-07-31:**\n- O004 → S003: 2026-02-06 ✓\n- O006 → S005: 2026-01-15 ✓\n- O007 → S006: NULL ✗\n- O009 → S008: NULL ✗\n- O013 → S012: 2025-08-17 ✓\n- O017 → S016: 2025-10-04 ✓\n- O021", "obj": {"answer": 12, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "C-118 HAL shipments with customer confirmation by 2026-07-31"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 1, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-12", "family": "COMP", "rep": 0, "latency_s": 25.65, "prompt_chars": 15105, "raw": "```json\n{\n  \"answer\": 139070,\n  \"definition_id\": \"meridian.def.order_value\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Sum of (gross_amount - discount_amount) for 15 non-cancelled orders placed 2026-05-01 to 2026-07-31.\"\n}\n```\n", "obj": {"answer": 139070, "definition_id": "meridian.def.order_value", "definition_version": 1, "unanswerable": false, "note": "Sum of (gross_amount - discount_amount) for 15 non-cancelled orders placed 2026-05-01 to 2026-07-31."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-09", "family": "XORG", "rep": 0, "latency_s": 13.26, "prompt_chars": 15232, "raw": "```json\n{\n  \"answer\": 9,\n  \"definition_id\": \"halden.def.business_day\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"9 business days: 2025-12-31, 2026-01-02, 01-05, 01-07 to 01-09, 01-12 to 01-14\"\n}\n```\n", "obj": {"answer": 9, "definition_id": "halden.def.business_day", "definition_version": 1, "unanswerable": false, "note": "9 business days: 2025-12-31, 2026-01-02, 01-05, 01-07 to 01-09, 01-12 to 01-14"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "COMP-02", "family": "COMP", "rep": 0, "latency_s": 53.3, "prompt_chars": 16461, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"180-day window: (2026-01-16, 2026-07-15]. Customers with non-cancelled orders in range.\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "180-day window: (2026-01-16, 2026-07-15]. Customers with non-cancelled orders in range."}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-08", "family": "COMP", "rep": 0, "latency_s": 41.57, "prompt_chars": 12541, "raw": "```json\n{\"answer\": 43, \"definition_id\": null, \"definition_version\": null, \"unanswerable\": false, \"note\": \"Delivered = customer_confirm_date not null and ≤ 2026-06-30\"}\n```\n", "obj": {"answer": 43, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "Delivered = customer_confirm_date not null and ≤ 2026-06-30"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "COMP-06", "family": "COMP", "rep": 1, "latency_s": 55.34, "prompt_chars": 16808, "raw": "```json\n{\n  \"answer\": 66043,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Invoices dated 2026-07-01 to 2026-08-31: gross minus discount plus freight; minus returns in period.\"\n}\n```\n", "obj": {"answer": 66043, "definition_id": "meridian.def.net_revenue", "definition_version": 2, "unanswerable": false, "note": "Invoices dated 2026-07-01 to 2026-08-31: gross minus discount plus freight; minus returns in period."}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-02", "family": "COMP", "rep": 1, "latency_s": 76.53, "prompt_chars": 15054, "raw": "```json\n{\n  \"answer\": 17,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Version 2 applies (2026-04-01 onwards, 180-day window)\"\n}\n```\n", "obj": {"answer": 17, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Version 2 applies (2026-04-01 onwards, 180-day window)"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose", "task": "COMP-09", "family": "COMP", "rep": 1, "latency_s": 57.45, "prompt_chars": 12541, "raw": "```json\n{\n  \"answer\": 37,\n  \"definition_id\": \"customer_confirmation_delivery\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Shipments with non-null customer_confirm_date on/before 2026-05-31\"\n}\n```\n", "obj": {"answer": 37, "definition_id": "customer_confirmation_delivery", "definition_version": 1, "unanswerable": false, "note": "Shipments with non-null customer_confirm_date on/before 2026-05-31"}, "parsed": true, "answer_correct": 1, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-07", "family": "TEMP", "rep": 1, "latency_s": 55.77, "prompt_chars": 16808, "raw": "```json\n{\n  \"answer\": 109343,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"13 invoices in period; 1 return; calculated per v1 formula\"\n}\n```\n", "obj": {"answer": 109343, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "13 invoices in period; 1 return; calculated per v1 formula"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-03", "family": "ABST", "rep": 1, "latency_s": 10.43, "prompt_chars": 16759, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"No damage status field in shipments table; unspecified returns reason.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "No damage status field in shipments table; unspecified returns reason."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "COMP-11", "family": "COMP", "rep": 0, "latency_s": 60.11, "prompt_chars": 15059, "raw": "```json\n{\"answer\": 21, \"definition_id\": \"meridian.def.on_time\", \"definition_version\": 2, \"unanswerable\": false, \"note\": \"21 shipments delivered by 2026-07-31 met on-time criteria\"}\n```\n", "obj": {"answer": 21, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "21 shipments delivered by 2026-07-31 met on-time criteria"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-01", "family": "ABST", "rep": 0, "latency_s": 151.77, "prompt_chars": 13956, "raw": "```json\n{\n  \"answer\": 6,\n  \"definition_id\": \"meridian-active-customers\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"Customers active March 31 but not June 30, 2026.\"\n}\n```\n", "obj": {"answer": 6, "definition_id": "meridian-active-customers", "definition_version": 1, "unanswerable": false, "note": "Customers active March 31 but not June 30, 2026."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_pad", "task": "ABST-06", "family": "ABST", "rep": 1, "latency_s": 54.13, "prompt_chars": 16431, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": true,\n  \"note\": \"Future date requires unknown orders between now and 2027-01-01\"\n}\n```\n", "obj": {"answer": null, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": true, "note": "Future date requires unknown orders between now and 2027-01-01"}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy", "task": "XORG-02", "family": "XORG", "rep": 0, "latency_s": 35.14, "prompt_chars": 15099, "raw": "```json\n{\n  \"answer\": 14,\n  \"definition_id\": \"halden.def.delivered\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"14 C-118 HAL shipments with carrier_scan_date on/before 2026-06-30\"\n}\n```\n", "obj": {"answer": 14, "definition_id": "halden.def.delivered", "definition_version": 2, "unanswerable": false, "note": "14 C-118 HAL shipments with carrier_scan_date on/before 2026-06-30"}, "parsed": true, "answer_correct": 0, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "ABST-06", "family": "ABST", "rep": 1, "latency_s": 79.59, "prompt_chars": 16761, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": \"meridian.def.active_customer\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"Customers C001, C004, C011, C013, C022 have non-cancelled orders in window (2026-07-05, 2027-01-01].\"\n}\n```\n", "obj": {"answer": 5, "definition_id": "meridian.def.active_customer", "definition_version": 2, "unanswerable": false, "note": "Customers C001, C004, C011, C013, C022 have non-cancelled orders in window (2026-07-05, 2027-01-01]."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-02", "family": "ABST", "rep": 1, "latency_s": 27.72, "prompt_chars": 13935, "raw": "```json\n{\n  \"answer\": null,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": true,\n  \"note\": \"Gross margin requires cost of goods sold; data unavailable.\"\n}\n```\n", "obj": {"answer": null, "definition_id": null, "definition_version": null, "unanswerable": true, "note": "Gross margin requires cost of goods sold; data unavailable."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 1, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-08", "family": "TEMP", "rep": 0, "latency_s": 51.04, "prompt_chars": 16808, "raw": "```json\n{\n  \"answer\": 115288,\n  \"definition_id\": \"meridian.def.net_revenue\",\n  \"definition_version\": 1,\n  \"unanswerable\": false,\n  \"note\": \"13 invoices in period; 1 return dated 2025-11-03\"\n}\n```\n", "obj": {"answer": 115288, "definition_id": "meridian.def.net_revenue", "definition_version": 1, "unanswerable": false, "note": "13 invoices in period; 1 return dated 2025-11-03"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "vercy_fed", "task": "TEMP-10", "family": "TEMP", "rep": 1, "latency_s": 58.07, "prompt_chars": 16796, "raw": "```json\n{\n  \"answer\": 12,\n  \"definition_id\": \"meridian.def.on_time\",\n  \"definition_version\": 2,\n  \"unanswerable\": false,\n  \"note\": \"12 delivered shipments as of 2026-02-01 were on time\"\n}\n```\n", "obj": {"answer": 12, "definition_id": "meridian.def.on_time", "definition_version": 2, "unanswerable": false, "note": "12 delivered shipments as of 2026-02-01 were on time"}, "parsed": true, "answer_correct": 1, "def_correct": 1, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 1, "scored_def": 1, "scored_abst": 0}
{"model": "claude-haiku-4-5-20251001", "arm": "prose_full", "task": "ABST-06", "family": "ABST", "rep": 0, "latency_s": 124.1, "prompt_chars": 13952, "raw": "```json\n{\n  \"answer\": 5,\n  \"definition_id\": null,\n  \"definition_version\": null,\n  \"unanswerable\": false,\n  \"note\": \"5 customers active: C001, C004, C011, C013, C022 with non-cancelled orders in 180-day window ending 2027-01-01.\"\n}\n```\n", "obj": {"answer": 5, "definition_id": null, "definition_version": null, "unanswerable": false, "note": "5 customers active: C001, C004, C011, C013, C022 with non-cancelled orders in 180-day window ending 2027-01-01."}, "parsed": true, "answer_correct": 0, "def_correct": 0, "abstained": 0, "used_alt_definition": 0, "false_abstention": 0, "scored_answer": 0, "scored_def": 0, "scored_abst": 1}