DerekSurge
94 lines
34 KiB
JSON
94 lines
34 KiB
JSON
[
|
|
{
|
|
"id": "0582f69f-6456-4e9f-a350-fc8a218496b6",
|
|
"sort_order": 0,
|
|
"rubric_text": "In file `quarantine_releases/REC-20260414-001_release.pdf`, the release PDF for REC-20260414-001 must exist in the quarantine_releases subfolder of the workspace.",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n pdf_path = Path(workspace_path) / 'quarantine_releases' / 'REC-20260414-001_release.pdf'\n if pdf_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': f'Found REC-20260414-001_release.pdf at {pdf_path}'}\n # Also check workspace root in case folder structure differs\n alt_path = Path(workspace_path) / 'REC-20260414-001_release.pdf'\n if alt_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': f'Found REC-20260414-001_release.pdf at {alt_path}'}\n return {'pass': False, 'score': 0.0, 'feedback': f'REC-20260414-001_release.pdf not found at {pdf_path} or {alt_path}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "6907a45e-d40b-4af4-af23-242e15476550",
|
|
"sort_order": 1,
|
|
"rubric_text": "In file `quarantine_releases/REC-20260414-002_release.pdf`, this file must NOT exist (request was not actionable).",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n pdf_path = Path(workspace_path) / 'quarantine_releases' / 'REC-20260414-002_release.pdf'\n alt_path = Path(workspace_path) / 'REC-20260414-002_release.pdf'\n if pdf_path.exists() or alt_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': f'REC-20260414-002_release.pdf should NOT exist (request was not actionable) but was found.'}\n return {'pass': True, 'score': 1.0, 'feedback': 'REC-20260414-002_release.pdf correctly does not exist.'}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "a773a42e-e1fe-4032-a1c8-31074ef3a0aa",
|
|
"sort_order": 2,
|
|
"rubric_text": "In file `inventory_master.xlsx`, row 8 must have: cell G8 = 285, cell M8 = 04/14/2026, cell N8 = 'REC-20260414-001', cell O8 = 0, cell P8 = blank, and cell S8 = 'HT-2026-0218, HT-2026-1803'.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).glob('inventory_master.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*inventory_master*'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'inventory_master.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(candidates[0], data_only=True)\n ws = wb.active\n issues = []\n\n # G8 = 285\n g8 = ws['G8'].value\n try:\n g8_num = float(g8) if g8 is not None else None\n if g8_num is None or abs(g8_num - 285) > 2.85:\n issues.append(f'G8 expected 285, got {g8}')\n except (TypeError, ValueError):\n issues.append(f'G8 expected 285, got {g8}')\n\n # M8 = 04/14/2026\n m8 = ws['M8'].value\n m8_ok = False\n if m8 is not None:\n if isinstance(m8, (datetime, date)):\n dt = m8 if isinstance(m8, datetime) else datetime(m8.year, m8.month, m8.day)\n if dt.month == 4 and dt.day == 14 and dt.year == 2026:\n m8_ok = True\n elif isinstance(m8, str):\n m8_str = m8.strip()\n if '04/14/2026' in m8_str or '4/14/2026' in m8_str or '2026-04-14' in m8_str:\n m8_ok = True\n if not m8_ok:\n issues.append(f'M8 expected 04/14/2026, got {m8}')\n\n # N8 = 'REC-20260414-001'\n n8 = ws['N8'].value\n if n8 is None or str(n8).strip() != 'REC-20260414-001':\n issues.append(f'N8 expected REC-20260414-001, got {n8}')\n\n # O8 = 0\n o8 = ws['O8'].value\n try:\n o8_num = float(o8) if o8 is not None else None\n if o8_num is None or abs(o8_num - 0) > 0.01:\n issues.append(f'O8 expected 0, got {o8}')\n except (TypeError, ValueError):\n issues.append(f'O8 expected 0, got {o8}')\n\n # P8 = blank\n p8 = ws['P8'].value\n if p8 is not None and str(p8).strip() != '':\n issues.append(f'P8 expected blank, got {p8}')\n\n # S8 = 'HT-2026-0218, HT-2026-1803'\n s8 = ws['S8'].value\n if s8 is None:\n issues.append(f'S8 expected HT-2026-0218, HT-2026-1803, got None')\n else:\n s8_str = str(s8).strip()\n if 'HT-2026-0218' not in s8_str or 'HT-2026-1803' not in s8_str:\n issues.append(f'S8 expected to contain HT-2026-0218 and HT-2026-1803, got {s8_str}')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'inventory_master.xlsx row 8 issues: ' + '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': f'inventory_master.xlsx row 8 verified: G8={g8}, M8={m8}, N8={n8}, O8={o8}, P8={p8}, S8={s8}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "633b2dbd-6618-480b-bee1-843a9ad68813",
|
|
"sort_order": 3,
|
|
"rubric_text": "In file `inventory_master.xlsx`, the spreadsheet must have exactly 20 data rows (last data row on row 21).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).glob('inventory_master.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*inventory_master*'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'inventory_master.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(candidates[0], data_only=True)\n ws = wb.active\n # Count non-empty rows starting from row 2 (assuming row 1 is header)\n data_rows = 0\n last_data_row = 1 # default to header row if no data\n for row_idx, row in enumerate(ws.iter_rows(min_row=2), start=2):\n if any(cell.value is not None and str(cell.value).strip() != '' for cell in row):\n data_rows += 1\n last_data_row = row_idx\n \n issues = []\n passed = True\n \n if data_rows != 20:\n passed = False\n issues.append(f'Expected exactly 20 data rows, found {data_rows} non-empty data rows.')\n \n if last_data_row != 21:\n passed = False\n issues.append(f'Expected last data row on row 21, but last non-empty data row is on row {last_data_row}.')\n \n if passed:\n return {'pass': True, 'score': 1.0, 'feedback': f'inventory_master.xlsx has exactly 20 data rows with last data row on row 21 as expected.'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'inventory_master.xlsx: ' + ' '.join(issues)}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "7056c0e4-718d-4ecf-b54b-abe1d1497bd2",
|
|
"sort_order": 4,
|
|
"rubric_text": "In file `receiving_log.xlsx`, row 10 must have: cell V10 = 'COC Received and Verified', cell W10 = 'Accepted', cell X10 = 'ACCEPTED', cell Z10 = date 04/23/2026 at or after 09:00 CST, cell AA10 = 'A-09-01', and cell AB10 = 'Putaway Complete'.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime, date, time\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).glob('receiving_log.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*receiving_log*'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'receiving_log.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(candidates[0], data_only=True)\n ws = wb.active\n issues = []\n\n # V10\n v10 = ws['V10'].value\n if v10 is None or 'COC Received and Verified'.lower() not in str(v10).lower():\n issues.append(f'V10 expected COC Received and Verified, got {v10}')\n\n # W10\n w10 = ws['W10'].value\n if w10 is None or 'accepted' not in str(w10).lower():\n issues.append(f'W10 expected Accepted, got {w10}')\n\n # X10\n x10 = ws['X10'].value\n if x10 is None or 'ACCEPTED' not in str(x10).upper():\n issues.append(f'X10 expected ACCEPTED, got {x10}')\n\n # Z10 = 04/23/2026 at or after 09:00\n z10 = ws['Z10'].value\n z10_ok = False\n if z10 is not None:\n if isinstance(z10, datetime):\n if z10.year == 2026 and z10.month == 4 and z10.day == 23:\n if z10.hour > 9 or (z10.hour == 9 and z10.minute >= 0):\n z10_ok = True\n elif isinstance(z10, date):\n if z10.year == 2026 and z10.month == 4 and z10.day == 23:\n z10_ok = True\n elif isinstance(z10, str):\n if '04/23/2026' in z10 or '4/23/2026' in z10 or '2026-04-23' in z10:\n z10_ok = True\n if not z10_ok:\n issues.append(f'Z10 expected 04/23/2026 at or after 09:00, got {z10}')\n\n # AA10\n aa10 = ws['AA10'].value\n if aa10 is None or str(aa10).strip().upper() != 'A-09-01':\n issues.append(f'AA10 expected A-09-01, got {aa10}')\n\n # AB10\n ab10 = ws['AB10'].value\n if ab10 is None or 'Putaway Complete'.lower() not in str(ab10).lower():\n issues.append(f'AB10 expected Putaway Complete, got {ab10}')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'receiving_log.xlsx row 10 issues: ' + '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': f'receiving_log.xlsx row 10 verified: V10={v10}, W10={w10}, X10={x10}, Z10={z10}, AA10={aa10}, AB10={ab10}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "0fde1f50-a8ba-49af-aa77-2d7ed55ccded",
|
|
"sort_order": 5,
|
|
"rubric_text": "In file `receiving_log.xlsx`, the spreadsheet must have exactly 12 data rows (last data row on row 13).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).glob('receiving_log.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*receiving_log*'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'receiving_log.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(candidates[0], data_only=True)\n ws = wb.active\n data_rows = 0\n for row in ws.iter_rows(min_row=2):\n if any(cell.value is not None and str(cell.value).strip() != '' for cell in row):\n data_rows += 1\n if data_rows == 12:\n return {'pass': True, 'score': 1.0, 'feedback': f'receiving_log.xlsx has exactly 12 data rows as expected.'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'receiving_log.xlsx expected 12 data rows, found {data_rows} non-empty data rows.'}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "9ed41faa-6e26-4966-b450-fb6f0739baa4",
|
|
"sort_order": 6,
|
|
"rubric_text": "In `ncr_log.xlsx`, cell L6 must remain 'Open' and cell L7 must remain 'Open' (no changes to the NCR log).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n candidates = list(Path(workspace_path).glob('ncr_log.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*ncr_log*'))\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'ncr_log.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(candidates[0], data_only=True)\n ws = wb.active\n issues = []\n l6 = ws['L6'].value\n if l6 is None or str(l6).strip().lower() != 'open':\n issues.append(f'L6 expected Open, got {l6}')\n l7 = ws['L7'].value\n if l7 is None or str(l7).strip().lower() != 'open':\n issues.append(f'L7 expected Open, got {l7}')\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'ncr_log.xlsx issues: ' + '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': f'ncr_log.xlsx L6={l6}, L7={l7} both remain Open as expected.'}",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "20df59e2-389b-4eec-b36d-eb9e8bbb32b5",
|
|
"sort_order": 7,
|
|
"rubric_text": "In `mailbox.json` (external services), there should not be any replies to message_id = <msg0026@gearsystems-meridian.com> in the Sent folder.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check mailbox.json for sent emails.'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found in external services path.'}\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n emails = data.get('emails', [])\n sent_emails = [e for e in emails if str(e.get('folder', '')).lower() == 'sent']\n\n target_message_id = '<msg0026@gearsystems-meridian.com>'\n\n # Look for any sent email that is a reply to the target message_id.\n # A reply could be indicated by:\n # - in_reply_to field matching the target message_id\n # - references field containing the target message_id\n # - reply_to_message_id field matching\n # We check multiple common field names to be generous.\n replies = []\n for e in sent_emails:\n in_reply_to = str(e.get('in_reply_to', '') or '')\n references = str(e.get('references', '') or '')\n reply_to_message_id = str(e.get('reply_to_message_id', '') or '')\n # Also check if subject starts with Re: and references the message somehow\n # Check body or headers for the message id as well\n headers = e.get('headers', {})\n header_in_reply_to = str(headers.get('In-Reply-To', '') or '') if isinstance(headers, dict) else ''\n header_references = str(headers.get('References', '') or '') if isinstance(headers, dict) else ''\n\n if (target_message_id in in_reply_to or\n target_message_id in references or\n target_message_id in reply_to_message_id or\n target_message_id in header_in_reply_to or\n target_message_id in header_references or\n # Also check without angle brackets\n 'msg0026@gearsystems-meridian.com' in in_reply_to or\n 'msg0026@gearsystems-meridian.com' in references or\n 'msg0026@gearsystems-meridian.com' in reply_to_message_id or\n 'msg0026@gearsystems-meridian.com' in header_in_reply_to or\n 'msg0026@gearsystems-meridian.com' in header_references):\n replies.append(e)\n\n if len(replies) == 0:\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'No replies to {target_message_id} found in the Sent folder. Total sent emails: {len(sent_emails)}. Check passed.'\n }\n else:\n reply_ids = [str(e.get('email_id', 'unknown')) for e in replies]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'Found {len(replies)} reply/replies to {target_message_id} in the Sent folder (email_ids: {reply_ids}). There should not be any replies to this message.'\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "3366695a-2408-4dc6-b59c-5e73ca53d99f",
|
|
"sort_order": 8,
|
|
"rubric_text": "Calendar event evt-025 must exist with title unchanged (no '[COMPLETED]'). Description must contain the original phrase 'Theo Brandt to verify COC against MTR heat numbers: HT-2026-1184, HT-2026-1185, HT-2026-0997 for receipt REC-20260414-001 (NCR-MER-2026-0247)' AND must contain a partial-release annotation including 'FG-91144-A1 released' with quantity and destination, and 'Remaining on Q-HOLD' listing both AX-04827-B2 and AX-05512-C1 with their quarantined quantities (44 and 96 respectively).",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check calendar_data.json.'}\n cal_path = Path(external_services_path) / 'calendar_data.json'\n if not cal_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'calendar_data.json not found in external services path.'}\n with open(cal_path, 'r') as f:\n data = json.load(f)\n events = data.get('events', {})\n if 'evt-025' not in events:\n return {'pass': False, 'score': 0.0, 'feedback': 'Event evt-025 is missing from calendar_data.json (may have been deleted).'}\n evt = events['evt-025']\n summary = evt.get('summary', '') or ''\n description = evt.get('description', '') or ''\n desc_lower = description.lower()\n issues = []\n\n # (1) Title must not contain [COMPLETED]\n if '[COMPLETED]' in summary.upper():\n issues.append(f'Title contains [COMPLETED]: {summary}')\n\n # (2) Description must contain original heat-number phrase\n heat_numbers = ['HT-2026-1184', 'HT-2026-1185', 'HT-2026-0997']\n for hn in heat_numbers:\n if hn not in description:\n issues.append(f'Description missing original heat number {hn}')\n break\n\n # (3) Description must contain partial-release annotation with FG-91144-A1 released\n has_partial_release = ('partial release' in desc_lower or 'partial-release' in desc_lower)\n has_fg_released = ('fg-91144-a1' in desc_lower and 'released' in desc_lower)\n if not has_partial_release:\n issues.append('Description missing Partial release annotation')\n if not has_fg_released:\n issues.append('Description missing FG-91144-A1 released')\n\n # (4) Description must contain Remaining on Q-HOLD with both AX-04827-B2 and AX-05512-C1\n has_remaining = ('remaining on q-hold' in desc_lower)\n if not has_remaining:\n issues.append('Description missing Remaining on Q-HOLD')\n if 'ax-04827-b2' not in desc_lower:\n issues.append('Description missing AX-04827-B2 in remaining quarantine list')\n if 'ax-05512-c1' not in desc_lower:\n issues.append('Description missing AX-05512-C1 in remaining quarantine list')\n\n # (5) Check that 44 and 96 appear in the annotation context\n # Find the partial release annotation portion\n partial_idx = desc_lower.find('partial release')\n if partial_idx == -1:\n partial_idx = desc_lower.find('partial-release')\n if partial_idx >= 0:\n annotation_text = desc_lower[partial_idx:]\n if '44' not in annotation_text:\n issues.append('Annotation missing quarantined quantity 44 for AX-04827-B2')\n if '96' not in annotation_text:\n issues.append('Annotation missing quarantined quantity 96 for AX-05512-C1')\n else:\n if '44' not in desc_lower or '96' not in desc_lower:\n issues.append('Description missing quarantined quantities 44 and/or 96')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'evt-025 issues: ' + '; '.join(issues) + f'. Description: {description[:500]}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Event evt-025 verified: title has no [COMPLETED], description contains original heat-number phrase, partial-release annotation with FG-91144-A1 released, Remaining on Q-HOLD listing AX-04827-B2 (44) and AX-05512-C1 (96).'}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "5de62afe-3687-426c-af6f-53d58dea3b01",
|
|
"sort_order": 9,
|
|
"rubric_text": "In slack_data.json (external services), the #receiving-dock channel must have the message containing 'QUARANTINE RELEASED: REC-20260414-001, SKU FG-91144-A1, qty 200 released from Q-HOLD to A-09-01. Authorization: Theo Brandt, 04/23/2026.'",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external_services_path provided; cannot check Slack data.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in external services path.'}\n with open(slack_path, 'r') as f:\n data = json.load(f)\n channels = data.get('channels', {})\n # Find the #receiving-dock channel ID\n dock_channel_id = None\n for cid, cinfo in channels.items():\n channel_name = cinfo.get('name', '').lower().replace('-', '').replace('_', '')\n if channel_name in ['receivingdock']:\n dock_channel_id = cid\n break\n if dock_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#receiving-dock channel not found in slack_data.json. Available channels: ' + str([c.get('name') for c in channels.values()])}\n messages_dict = data.get('messages', {})\n msgs = messages_dict.get(dock_channel_id, [])\n if not msgs:\n return {'pass': False, 'score': 0.0, 'feedback': '#receiving-dock channel has no messages.'}\n\n # Required wording (whitespace-insensitive match) and identifiers (must appear exactly)\n required_text = 'QUARANTINE RELEASED: REC-20260414-001, SKU FG-91144-A1, qty 200 released from Q-HOLD to A-09-01. Authorization: Theo Brandt, 04/23/2026.'\n required_exact_tokens = ['REC-20260414-001', 'FG-91144-A1', 'A-09-01']\n\n def normalize_ws(s):\n # Collapse any run of whitespace (spaces, tabs, newlines) into a single space, strip ends\n return re.sub(r'\\s+', ' ', s).strip()\n\n required_norm = normalize_ws(required_text)\n\n # Check all messages in the channel for the required text (whitespace-insensitive)\n found = False\n found_text = ''\n for msg in msgs:\n msg_text = msg.get('text', '') or ''\n msg_norm = normalize_ws(msg_text)\n if required_norm in msg_norm:\n # Also require the three identifiers to appear exactly in the original message text\n missing = [tok for tok in required_exact_tokens if tok not in msg_text]\n if missing:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'#receiving-dock message matches the wording but is missing exact identifier(s): {missing}. Message: \"{msg_text[:300]}\"'}\n found = True\n found_text = msg_text\n break\n\n # Case-insensitive fallback for leniency (still whitespace-insensitive, still requires exact identifiers)\n if not found:\n for msg in msgs:\n msg_text = msg.get('text', '') or ''\n msg_norm = normalize_ws(msg_text)\n if required_norm.lower() in msg_norm.lower():\n missing = [tok for tok in required_exact_tokens if tok not in msg_text]\n if missing:\n return {'pass': False, 'score': 0.0,\n 'feedback': f'#receiving-dock message matches the wording (case-insensitive) but is missing exact identifier(s): {missing}. Message: \"{msg_text[:300]}\"'}\n found = True\n found_text = msg_text\n break\n\n if not found:\n sorted_msgs = sorted(msgs, key=lambda m: float(m.get('ts', 0)))\n recent_texts = [m.get('text', '')[:200] for m in sorted_msgs[-3:]]\n return {'pass': False, 'score': 0.0,\n 'feedback': f'#receiving-dock channel does not contain required quarantine release message. Required text (whitespace-insensitive): \"{required_text}\". Last {len(recent_texts)} messages: {recent_texts}'}\n\n return {'pass': True, 'score': 1.0, 'feedback': f'#receiving-dock channel contains the required quarantine release message: {found_text[:300]}'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1776321019779",
|
|
"sort_order": 10,
|
|
"rubric_text": "In file `inventory_master.xlsx`, cell O11 must be \"144\", P11 must be \"Quarantine - Tier 2 damage pending inspection (REC-20260414-002)\", O19 must be \"44\", P19 must be \"QUARANTINE — missing COC (REC-20260414-001)\", O21 must be \"144\", and P21 must be \"QUARANTINE — Tier 2 damage pending inspection (REC-20260414-002)\".",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find inventory_master.xlsx\n file_path = Path(workspace_path) / \"inventory_master.xlsx\"\n \n if not file_path.exists():\n xlsx_files = list(Path(workspace_path).glob(\"**/inventory_master.xlsx\"))\n if xlsx_files:\n file_path = xlsx_files[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File `inventory_master.xlsx` not found in workspace.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open `inventory_master.xlsx`: {e}\"}\n \n ws = wb.active\n if ws is None:\n ws = wb.worksheets[0] if wb.worksheets else None\n \n if ws is None:\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No worksheets found in `inventory_master.xlsx`.\"}\n \n feedback_parts = []\n checks_passed = 0\n total_checks = 6\n \n def cell_str(row, col):\n val = ws.cell(row=row, column=col).value\n if val is None:\n return None\n return str(val).strip()\n \n def normalize_dash(s):\n if s is None:\n return None\n return s.replace('\\u2014', '-').replace('\\u2013', '-').replace('--', '-').lower().strip()\n \n def check_numeric(row, col, expected, label):\n nonlocal checks_passed\n val = cell_str(row, col)\n # Accept numeric equivalence: \"144\" or \"144.0\" etc.\n match = False\n if val is not None:\n try:\n if float(val) == float(expected):\n match = True\n except ValueError:\n pass\n if val == expected:\n match = True\n if match:\n checks_passed += 1\n feedback_parts.append(f\"PASS: {label} = '{val}' (expected '{expected}')\")\n else:\n feedback_parts.append(f\"FAIL: {label} = '{val}' (expected '{expected}')\")\n \n def check_text(row, col, expected, label):\n nonlocal checks_passed\n val = cell_str(row, col)\n if val is not None and normalize_dash(val) == normalize_dash(expected):\n checks_passed += 1\n feedback_parts.append(f\"PASS: {label} = '{val}'\")\n else:\n feedback_parts.append(f\"FAIL: {label} = '{val}' (expected '{expected}')\")\n \n # O = column 15, P = column 16\n \n # Check O11 must be \"144\"\n check_numeric(11, 15, \"144\", \"O11\")\n \n # Check P11 must be \"Quarantine - Tier 2 damage pending inspection (REC-20260414-002)\"\n check_text(11, 16, \"Quarantine - Tier 2 damage pending inspection (REC-20260414-002)\", \"P11\")\n \n # Check O19 must be \"44\"\n check_numeric(19, 15, \"44\", \"O19\")\n \n # Check P19 must be \"QUARANTINE \\u2014 missing COC (REC-20260414-001)\"\n check_text(19, 16, \"QUARANTINE \\u2014 missing COC (REC-20260414-001)\", \"P19\")\n \n # Check O21 must be \"144\"\n check_numeric(21, 15, \"144\", \"O21\")\n \n # Check P21 must be \"QUARANTINE \\u2014 Tier 2 damage pending inspection (REC-20260414-002)\"\n check_text(21, 16, \"QUARANTINE \\u2014 Tier 2 damage pending inspection (REC-20260414-002)\", \"P21\")\n \n wb.close()\n \n score = checks_passed / total_checks\n passed = checks_passed == total_checks\n \n summary = f\"{checks_passed}/{total_checks} checks passed. \" + \" | \".join(feedback_parts)\n \n return {\"pass\": passed, \"score\": score, \"feedback\": summary}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776327435121",
|
|
"sort_order": 11,
|
|
"rubric_text": "In file `quarantine_releases/REC-20260414-001_release.pdf`, the content must include the following text: \"I've reviewed the COC submitted by Lakewood Precision Forgings this morning for the FG-91144-A1 line under REC-20260414-001.\", \"The COC is acceptable\", \"heat number HT-2026-1803 is confirmed and the documentation is complete.\", \"You are authorized to release FG-91144-A1 from Q-HOLD. Quantity: 200 units. Please proceed with the standard quarantine release procedure.\", and \"Theo Brandt\".",
|
|
"verifier_code": "from pathlib import Path\nimport re\n\n\ndef verify(workspace_path, external_services_path=None):\n \"\"\"\n Verify that quarantine_releases/REC-20260414-001_release.pdf exists and\n contains the required text phrases from the rubric.\n \"\"\"\n workspace_path = Path(workspace_path)\n pdf_path = workspace_path / \"quarantine_releases\" / \"REC-20260414-001_release.pdf\"\n\n if not pdf_path.exists():\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"PDF file not found at {pdf_path}\",\n }\n\n # ---- Extract PDF text ----\n pdf_text = \"\"\n pdf_errors = []\n try:\n import pdfplumber\n with pdfplumber.open(pdf_path) as pdf:\n for page in pdf.pages:\n t = page.extract_text()\n if t:\n pdf_text += t + \"\\n\"\n except Exception as e:\n pdf_errors.append(f\"pdfplumber: {e}\")\n\n if not pdf_text.strip():\n try:\n import PyPDF2\n with open(pdf_path, \"rb\") as f:\n reader = PyPDF2.PdfReader(f)\n for page in reader.pages:\n t = page.extract_text()\n if t:\n pdf_text += t + \"\\n\"\n except Exception as e:\n pdf_errors.append(f\"PyPDF2: {e}\")\n\n if not pdf_text.strip():\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not extract text from PDF at {pdf_path}. Errors: {pdf_errors}\",\n }\n\n # Normalize whitespace for matching (collapse all whitespace to single space)\n pdf_text_normalized = re.sub(r\"\\s+\", \" \", pdf_text)\n\n # ---- Required phrases from the rubric ----\n required_phrases = [\n \"I've reviewed the COC submitted by Lakewood Precision Forgings this morning for the FG-91144-A1 line under REC-20260414-001.\",\n \"The COC is acceptable\",\n \"heat number HT-2026-1803 is confirmed and the documentation is complete.\",\n \"You are authorized to release FG-91144-A1 from Q-HOLD. Quantity: 200 units. Please proceed with the standard quarantine release procedure.\",\n \"Theo Brandt\",\n ]\n\n found_phrases = []\n missing_phrases = []\n\n for phrase in required_phrases:\n # Normalize the phrase the same way\n phrase_normalized = re.sub(r\"\\s+\", \" \", phrase)\n # Case-insensitive check\n if phrase_normalized.lower() in pdf_text_normalized.lower():\n found_phrases.append(phrase)\n else:\n # Try a more lenient check: compare significant words\n # Extract words >= 3 chars from the phrase\n phrase_words = [w.lower().strip(\".,;:!?()'\\\"\\u2014-\") for w in re.split(r\"\\s+\", phrase_normalized)]\n phrase_words = [w for w in phrase_words if len(w) >= 3]\n if phrase_words:\n pdf_lower = pdf_text_normalized.lower()\n words_found = sum(1 for w in phrase_words if w in pdf_lower)\n ratio = words_found / len(phrase_words)\n if ratio >= 0.9:\n found_phrases.append(phrase)\n else:\n missing_phrases.append(f\"{phrase!r} (word match: {words_found}/{len(phrase_words)} = {ratio:.0%})\")\n else:\n missing_phrases.append(phrase)\n\n total = len(required_phrases)\n passed_count = len(found_phrases)\n score = passed_count / total if total > 0 else 0.0\n passed = passed_count == total\n\n if passed:\n feedback = (\n f\"PDF at {pdf_path} contains all {total} required phrases. \"\n f\"All checks passed.\"\n )\n else:\n feedback = (\n f\"PDF at {pdf_path} is missing {len(missing_phrases)}/{total} required phrases. \"\n f\"Missing: {'; '.join(missing_phrases)}\"\n )\n\n return {\"pass\": passed, \"score\": score, \"feedback\": feedback}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1776897621818",
|
|
"sort_order": 12,
|
|
"rubric_text": "In file `receiving_log.xlsx`, the Status column for rows 8 and 9 (cells AB8 and AB9) should contain the exact text \"Quarantined\".",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'receiving_log.xlsx' not found in workspace at {workspace_path}\"}\n \n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening 'receiving_log.xlsx': {e}\"}\n \n results = {}\n cells_to_check = {\"AB8\": 8, \"AB9\": 9}\n \n for cell_ref, row_num in cells_to_check.items():\n cell_value = ws[cell_ref].value\n if cell_value is not None:\n cell_str = str(cell_value).strip()\n else:\n cell_str = None\n results[cell_ref] = cell_str\n \n wb.close()\n \n passed = True\n feedback_parts = []\n \n for cell_ref, actual_value in results.items():\n if actual_value is not None and actual_value.lower() == \"quarantined\":\n feedback_parts.append(f\"{cell_ref} = '{actual_value}' ✓\")\n else:\n passed = False\n feedback_parts.append(f\"{cell_ref} = '{actual_value}' (expected 'Quarantined') ✗\")\n \n score = sum(1 for cell_ref, val in results.items() if val is not None and val.lower() == \"quarantined\") / len(cells_to_check)\n \n feedback = \"Checking receiving_log.xlsx cells AB8 and AB9 for 'Quarantined': \" + \"; \".join(feedback_parts)\n \n return {\"pass\": passed, \"score\": score, \"feedback\": feedback}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
}
|
|
]
|