31 lines
25 KiB
JSON
31 lines
25 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "09997602-ff2f-463b-b98e-eda0b08edda0",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In file 'inbound_receiving_log.xlsx', row 17 columns A–O read: 2026-04-02 | 2026-04-02-001 | Waco Transfer | 20 | 400 | -9 | -10 | -13 | -9 | Tier 2 | 399 | -1 | -0.25 | One case label torn (SC-001 pallet 3). No breach. | Accept with Note |; column P contains \"Pending Variance Resolution\"; and column Q contains \"Accepted – Warm side of acceptable range\", \"Class 1 damage\", and \"Minor variance\"",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\nCOLS = {\n 'date': 0,\n 'shipment': 1,\n 'origin': 2,\n 'pallets': 3,\n 'exp_cases': 4,\n 'temp1': 5,\n 'temp2': 6,\n 'temp3': 7,\n 'warmest': 8,\n 'tier': 9,\n 'actual': 10,\n 'variance': 11,\n 'var_pct': 12,\n 'damage': 13,\n 'disposition': 14,\n 'status': 15,\n 'notes': 16,\n}\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n if not xlsx_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No xlsx file found in workspace'}\n\n def norm(c):\n if c is None:\n return ''\n if isinstance(c, (datetime, date)):\n return str(c.date() if isinstance(c, datetime) else c)\n s = str(c).strip().lower()\n for ch in ('\\u2212', '\\u2013', '\\u2014'):\n s = s.replace(ch, '-')\n s = re.sub(r'[^\\w\\s.\\-]', '', s)\n s = re.sub(r'\\s+', ' ', s).strip()\n return s\n\n best_result = None\n\n for xlsx_file in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_file, data_only=True)\n except Exception:\n continue\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows(values_only=True))\n\n candidate_rows = []\n if len(rows) >= 17:\n candidate_rows.append((16, rows[16]))\n for idx, r in enumerate(rows):\n if idx == 16:\n continue\n r_str = ' '.join([str(x) for x in r if x is not None]).lower()\n if '2026-04-02-001' in r_str or ('waco' in r_str and '2026-04-02' in r_str):\n candidate_rows.append((idx, r))\n\n if not candidate_rows:\n result = {'pass': False, 'score': 0.0, 'feedback': f'{xlsx_file.name}/{sheet_name}: no row found for this shipment'}\n if best_result is None or result['score'] > best_result['score']:\n best_result = result\n continue\n\n for row_idx, row in candidate_rows:\n row = list(row) + [''] * max(0, 17 - len(row))\n\n def ceq(col, *targets):\n return norm(row[col]) in [norm(t) for t in targets]\n\n def csub(col, target):\n return norm(target) in norm(row[col])\n\n checks = {\n 'date 2026-04-02': (COLS['date'], csub(COLS['date'], '2026-04-02'), '2026-04-02'),\n 'shipment 2026-04-02-001': (COLS['shipment'], csub(COLS['shipment'], '2026-04-02-001'), '2026-04-02-001'),\n 'waco transfer': (COLS['origin'], ceq(COLS['origin'], 'waco transfer'), 'Waco Transfer'),\n 'pallets 20': (COLS['pallets'], ceq(COLS['pallets'], '20', '20.0'), '20'),\n 'expected cases 400': (COLS['exp_cases'], ceq(COLS['exp_cases'], '400', '400.0'), '400'),\n 'temp 1 -9': (COLS['temp1'], ceq(COLS['temp1'], '-9', '-9.0'), '-9'),\n 'temp 2 -10': (COLS['temp2'], ceq(COLS['temp2'], '-10', '-10.0'), '-10'),\n 'temp 3 -13': (COLS['temp3'], ceq(COLS['temp3'], '-13', '-13.0
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "ec283121-a288-4a9a-b212-60f83d94f5f0",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In `slack_data.json`, a message in C001 (receiving-houston): \"Variance detected. Manifest 2026-04-02-001 received with flags. Temp: -9°F (P4). Variance: -1 case. Email sent to the Production Planner and DC Manager.\"",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n\n with open(slack_path) as f:\n data = json.load(f)\n\n channels = data.get('channels', {})\n channel_id = None\n if 'C001' in channels:\n channel_id = 'C001'\n else:\n for cid, cinfo in channels.items():\n cname = cinfo.get('name', '').lower().replace('-', '').replace('_', '').replace(' ', '')\n if 'receivinghouston' in cname:\n channel_id = cid\n break\n\n if channel_id is None:\n return {\n 'pass': False, 'score': 0.0,\n 'feedback': f'C001 / #receiving-houston not found. Available: {list(channels.keys())}'\n }\n\n messages = data.get('messages', {}).get(channel_id, [])\n if not messages:\n return {'pass': False, 'score': 0.0, 'feedback': f'No messages in channel {channel_id}'}\n \n INITIAL_LAST_TS = 1775137500.0\n messages = [m for m in messages if float(m.get('ts', 0)) > INITIAL_LAST_TS]\n if not messages:\n return {'pass': False, 'score': 0.0, 'feedback': f'No new messages in C001 (#receiving-houston) after ts {INITIAL_LAST_TS}'}\n\n def norm(s):\n s = str(s).lower()\n for ch in ('\\u2212', '\\u2013', '\\u2014'):\n s = s.replace(ch, '-')\n s = re.sub(r'[^\\w\\s\\-]', '', s) # removes °, parens, colon, comma — keeps hyphens\n s = re.sub(r'\\s+', ' ', s).strip()\n return s\n\n # S3 template components — all required\n CHECKS = {\n 'opening phrase': lambda t: 'variance detected' in t,\n 'manifest': lambda t: '2026-04-02-001' in t,\n 'received with flags': lambda t: 'received with flags' in t,\n 'temp -9': lambda t: '-9' in t,\n 'temp tier p4': lambda t: 'p4' in t,\n 'variance -1 case': lambda t: '-1' in t and 'case' in t,\n 'email sent': lambda t: 'email sent' in t,\n 'production planner': lambda t: 'production planner' in t,\n 'dc manager': lambda t: 'dc manager' in t\n }\n\n matching = []\n best_score = 0.0\n best_detail = None\n\n for msg in messages:\n text = msg.get('text', '')\n t = norm(text)\n results = {k: fn(t) for k, fn in CHECKS.items()}\n failed = [k for k, v in results.items() if not v]\n score = (len(CHECKS) - len(failed)) / len(CHECKS)\n if score > best_score:\n best_score = score\n best_detail = {'failed': failed, 'text': text}\n if not failed:\n matching.append(msg)\n\n if len(matching) == 0:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'No S3 message found in C001 (#receiving-houston). '\n f'{len(messages)} message(s) checked. '\n f'Best match (score {best_score:.0%}) — failed: {best_detail[\"failed\"]}. '\n f'Text: \"{best_detail[\"text\"]}\"'\n )\n }\n if len(matching) > 1:\n return {\n 'pass': False,\n 'score': 0.5,\n 'feedback': (\n f'{len(matching)} matching messages found in C001; expected exactly 1. '\n f'Texts: {[m[\"text\"][:150] for m in matching]}'\n )\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Exactly 1 S3 message verified in C001 (#receiving-houston): \"{matching[0][\"text\"]}\"'\n }\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "ae36c51e-a489-4c45-934d-98a26c0e2041",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In `mailbox.json`, - one sent email:\n - To: Production Planner ([jacob.patel@prairiestar.com](mailto:jacob.patel@prairiestar.com))\n - CC: DC Manager ([carlos.mendoza@prairiestar.com](mailto:carlos.mendoza@prairiestar.com))\n - Subject: P3 – Variance Notification – 2026-04-02-001 – 2026-04-02\n - Body:\n \n Manifest: 2026-04-02-001\n SKU affected: SC-003\n Manifest qty: 120 cases\n Actual qty: 119 cases\n Overall Variance: -1 case (-0.25%)\n Classification: Minor\n \n Pattern escalation (third minor variance from Waco in 30 days)\n \n Please provide explanation or corrective action.\n \n Inventory Operations Analyst,\n Prairie Star Creamery",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\nINITIAL_LAST_EMAIL_ID = 11\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path is None'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n with open(mailbox_path) as f:\n data = json.load(f)\n\n def norm(s):\n s = str(s).lower()\n for ch in ('\\u2212', '\\u2013', '\\u2014'):\n s = s.replace(ch, '-')\n s = re.sub(r'[^\\w\\s\\-\\.]', '', s)\n s = re.sub(r'\\s+', ' ', s).strip()\n return s\n\n emails = data.get('emails', [])\n new_sent = [\n e for e in emails\n if e.get('folder', '').lower() == 'sent'\n and int(e.get('email_id', 0)) > INITIAL_LAST_EMAIL_ID\n ]\n\n if not new_sent:\n return {\n 'pass': False, 'score': 0.0,\n 'feedback': f'No new sent emails found (expected email_id > {INITIAL_LAST_EMAIL_ID})'\n }\n\n matching = []\n best_score = 0.0\n best_detail = None\n\n for email in new_sent:\n to_addr = (email.get('to_addr') or '').lower()\n cc_addr = (email.get('cc_addr') or '').lower()\n subj = norm(email.get('subject') or '')\n body = norm(email.get('body_text') or '')\n\n checks = {\n 'to jacob.patel': 'jacob.patel@prairiestar.com' in to_addr,\n 'cc carlos.mendoza': 'carlos.mendoza@prairiestar.com' in cc_addr,\n 'subject p3': 'p3' in subj,\n 'subject variance notification': 'variance notification' in subj,\n 'subject manifest': '2026-04-02-001' in subj,\n 'subject date': '2026-04-02' in subj,\n 'body manifest': 'manifest 2026-04-02-001' in body,\n 'body sku sc-003': 'sc-003' in body,\n 'body manifest qty 120': 'manifest qty' in body and '120' in body,\n 'body actual qty 119': 'actual qty' in body and '119' in body,\n 'body overall variance': 'overall variance' in body,\n 'body -1 case': '-1 case' in body,\n 'body -0.25%': '0.25' in body,\n 'body classification minor': 'classification' in body and 'minor' in body,\n 'body pattern escalation waco': 'pattern escalation' in body and 'waco' in body,\n 'body corrective action': 'explanation or corrective action' in body,\n 'body signature': 'inventory operations analyst' in body,\n }\n\n failed = [k for k, v in checks.items() if not v]\n score = (len(checks) - len(failed)) / len(checks)\n\n if score > best_score:\n best_score = score\n best_detail = {\n 'failed': failed,\n 'subject': email.get('subject', ''),\n 'to': to_addr,\n 'cc': cc_addr,\n 'body': (email.get('body_text') or '')[:400],\n }\n\n if not failed:\n matching.append(email)\n\n if len(matching) == 0:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': (\n f'No matching email found. {len(new_sent)} new sent email(s) checked. '\n f'Best match (score {best_score:.0%}) — failed: {best_detail[\"failed\"]}. '\n f'Subject: \"{best_detail[\"subject\"]}\", To: {best_detail[\"to\"]}, CC: {best_detail[\"cc\"]}. '\n f'Body: \"{best_detail[\"body\"]}\"'\n )\n }\n if len(matching) > 1:\n return {\n 'pass': False,\n 'score': 0.5,\n 'feedback': (\n f'{len(matching)} matching emails found; expecte
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776789196935",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "- `calendar_data.json`: has exactly 53 events.\n- `expiry_tracker.xlsx`: has exactly 16 data rows.\n- `inventory_master.xlsx`: has exactly 16 data rows.\n- `inbound_receiving_log.xlsx`: has exactly 14 data rows.\n- `mailbox.json`: has exactly 8 emails in the Sent folder, 0 in Drafts, 3 in Receiving, and 1 in Inbox.\n- `slack_data.json`: the following channels have exactly this many messages:\n - C001: 8\n - C002: 4\n - C003: 0\n - C004: 0\n - C005: 0\n - C006: 1\n - D001: 1\n - D002: 0\n - D003: 0\n - D004: 0",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n failures = []\n checks_passed = 0\n total_checks = 0\n\n def count_xlsx_data_rows(file_path):\n wb = openpyxl.load_workbook(file_path, read_only=True, data_only=True)\n ws = wb.active\n rows = list(ws.iter_rows(min_row=1))\n wb.close()\n # All workspace xlsx files have 3 header rows (title, subtitle, column headers)\n data_rows = sum(\n 1 for row in rows[3:]\n if any(cell.value is not None for cell in row)\n )\n return data_rows\n\n # ── 1. calendar_data.json ──────────────────────────────────────────────────\n total_checks += 1\n cal_path = Path(external_services_path) / 'calendar_data.json' if external_services_path else None\n if cal_path and cal_path.exists():\n try:\n with open(cal_path) as f:\n cal = json.load(f)\n events = cal.get('events', {})\n n = len(events)\n if n == 53:\n checks_passed += 1\n else:\n failures.append(f'calendar_data.json: {n} events (expected 53)')\n except Exception as e:\n failures.append(f'calendar_data.json: read error — {e}')\n else:\n failures.append('calendar_data.json: not found')\n\n # ── 2. expiry_tracker.xlsx ────────────────────────────────────────────────\n total_checks += 1\n et_path = Path(workspace_path) / 'expiry_tracker.xlsx'\n if et_path.exists():\n try:\n n = count_xlsx_data_rows(et_path)\n if n == 16:\n checks_passed += 1\n else:\n failures.append(f'expiry_tracker.xlsx: {n} data rows (expected 16)')\n except Exception as e:\n failures.append(f'expiry_tracker.xlsx: read error — {e}')\n else:\n failures.append('expiry_tracker.xlsx: not found')\n\n # ── 3. inventory_master.xlsx ──────────────────────────────────────────────\n total_checks += 1\n im_path = Path(workspace_path) / 'inventory_master.xlsx'\n if im_path.exists():\n try:\n n = count_xlsx_data_rows(im_path)\n if n == 16:\n checks_passed += 1\n else:\n failures.append(f'inventory_master.xlsx: {n} data rows (expected 16)')\n except Exception as e:\n failures.append(f'inventory_master.xlsx: read error — {e}')\n else:\n failures.append('inventory_master.xlsx: not found')\n\n # ── 4. inbound_receiving_log.xlsx ─────────────────────────────────────────\n total_checks += 1\n irl_path = Path(workspace_path) / 'inbound_receiving_log.xlsx'\n if irl_path.exists():\n try:\n n = count_xlsx_data_rows(irl_path)\n if n == 14:\n checks_passed += 1\n else:\n failures.append(f'inbound_receiving_log.xlsx: {n} data rows (expected 14)')\n except Exception as e:\n failures.append(f'inbound_receiving_log.xlsx: read error — {e}')\n else:\n failures.append('inbound_receiving_log.xlsx: not found')\n\n # ── 5. mailbox.json ───────────────────────────────────────────────────────\n total_checks += 1\n mail_path = Path(external_services_path) / 'mailbox.json' if external_services_path else None\n if mail_path and mail_path.exists():\n try:\n with open(mail_path) as f:\n mail = json.load(f)\n folder
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
}
|
|||
|
|
]
|