108 lines
77 KiB
JSON
108 lines
77 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "19141bc5-78ca-4c87-b3f4-52e84c8c52e3",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), a sent email to daniel.reyes@prairiestar.com must exist that CCs carlos.mendoza@prairiestar.com and jacob.patel@prairiestar.com, references manifest number '2026-04-15-003', references priority level 'P2', mentions variance, and has attachment that contains 'IR-002-2026-04-15' and 'Variance' in its name and has the extension '.docx' - this name is not case-sensitive.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n best_feedback = \"No sent emails found at all.\"\n for email in emails:\n folder = (email.get(\"folder\") or \"\").lower()\n if folder not in [\"sent\", \"outbox\"]:\n continue\n # Check to address - be loose, check multiple field names\n to_addr = \"\"\n for key in [\"to_addr\", \"to\", \"to_address\", \"recipient\", \"recipients\"]:\n val = email.get(key)\n if val:\n if isinstance(val, list):\n to_addr = \" \".join([str(v) for v in val]).lower()\n else:\n to_addr = str(val).lower()\n break\n if \"daniel.reyes@prairiestar.com\" not in to_addr:\n best_feedback = f\"Found sent email but not addressed to daniel.reyes@prairiestar.com. To field: {to_addr}\"\n continue\n # Check CC - be loose, check multiple field names\n cc_addr = \"\"\n for key in [\"cc_addr\", \"cc\", \"cc_address\", \"CC\"]:\n val = email.get(key)\n if val:\n if isinstance(val, list):\n cc_addr = \" \".join([str(v) for v in val]).lower()\n else:\n cc_addr = str(val).lower()\n break\n missing_cc = []\n if \"carlos.mendoza@prairiestar.com\" not in cc_addr:\n missing_cc.append(\"carlos.mendoza@prairiestar.com\")\n if \"jacob.patel@prairiestar.com\" not in cc_addr:\n missing_cc.append(\"jacob.patel@prairiestar.com\")\n if missing_cc:\n best_feedback = f\"Found sent email to daniel.reyes but missing CC: {', '.join(missing_cc)}. CC field contained: {cc_addr}\"\n continue\n # Combine body and subject for content checks\n body = (email.get(\"body_text\") or email.get(\"body\") or email.get(\"content\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n combined = body + \" \" + subject\n # Check manifest reference\n if \"2026-04-15-003\" not in combined:\n best_feedback = \"Found sent email to daniel.reyes with correct CCs but missing manifest reference '2026-04-15-003'\"\n continue\n # Check for P2 reference - be very loose\n p2_found = bool(re.search(r'p[\\-\\s]?2', combined)) or 'priority 2' in combined or 'priority level 2' in combined or 'priority: 2' in combined\n if not p2_found:\n best_feedback = \"Found sent email to daniel.reyes with correct CCs and manifest, but missing priority level P2 reference\"\n continue\n # Check for variance mention - be loose\n if \"variance\" not in combined and \"discrepan\" not in combined:\n best_feedback = \"Found sent email to daniel.reyes with correct CCs, manifest and P2, but missing mention of variance\"\n continue\n # Check attachment - the rubric says the name must contain 'IR-002-2026-04-15' and 'Variance',\n # and have the extension '.docx' (case-insensitive)\n attachments = email.get(\"attachments\") or []\n attachment_names = []\n for a in attachments:\n if isinstance(a, str):\n attachment_names.append(a)\n elif isinstance(a, dict):\n for akey in [\"filename\", \"name\", \"file_name\", \"fileName\"]:\n aname = a.get(akey)\n if aname:\n attachment_names.appe
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "d4f6d70b-9d3a-415c-8cac-3af459212895",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), a sent email to daniel.reyes@prairiestar.com must exist that CCs carlos.mendoza@prairiestar.com and jacob.patel@prairiestar.com, references manifest number '2026-04-15-002', mentions the order being short 6 cases of SC-006 and having 8 extra cases of SC-008, requests immediate investigation, and has attachment 'IR-001-2026-04-15-Variance.docx'.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n candidates = []\n for email in emails:\n if email.get(\"folder\", \"\").lower() not in [\"sent\"]:\n continue\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n if \"daniel.reyes@prairiestar.com\" not in to_addr:\n continue\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n if \"carlos.mendoza@prairiestar.com\" not in cc_addr:\n continue\n if \"jacob.patel@prairiestar.com\" not in cc_addr:\n continue\n body = (email.get(\"body_text\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n combined = body + \" \" + subject\n if \"2026-04-15-002\" not in combined:\n continue\n if \"sc-006\" not in combined:\n continue\n if \"sc-008\" not in combined:\n continue\n # Check for short 6 cases of SC-006\n has_short_6 = bool(re.search(r'(short|missing|under).*6.*sc-006|sc-006.*short.*6|6.*case.*sc-006|sc-006.*6.*case', combined))\n has_sc006_shortage = '6' in combined and 'sc-006' in combined and any(w in combined for w in ['short', 'missing', 'under', 'discrepan'])\n if not (has_short_6 or has_sc006_shortage):\n continue\n # Check for 8 extra SC-008\n has_extra_8 = bool(re.search(r'(extra|over|excess|surplus).*8.*sc-008|sc-008.*(extra|over|excess|surplus).*8|8.*case.*sc-008|sc-008.*8.*case', combined))\n has_sc008_overage = '8' in combined and 'sc-008' in combined and any(w in combined for w in ['extra', 'over', 'excess', 'surplus', 'discrepan'])\n if not (has_extra_8 or has_sc008_overage):\n continue\n # Check for attachment - rubric says 'IR-001-2026-04-15-Variance.docx'\n attachments = email.get(\"attachments\") or []\n attachment_names = []\n for a in attachments:\n if isinstance(a, str):\n attachment_names.append(a.lower())\n elif isinstance(a, dict):\n name = str(a.get(\"filename\", \"\") or a.get(\"name\", \"\") or \"\").lower()\n attachment_names.append(name)\n # Accept IR-001-2026-04-15-Variance (the rubric specifies this) but also be lenient\n # Accept any attachment containing 'ir-' and '2026-04-15' and 'variance'\n has_attachment = any(\"ir-001-2026-04-15-variance\" in a for a in attachment_names)\n # Also accept looser match: any IR variance report for this date\n has_attachment_loose = any(\"variance\" in a and \"2026-04-15\" in a for a in attachment_names)\n if not (has_attachment or has_attachment_loose):\n continue\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found sent email for manifest 2026-04-15-002 to daniel.reyes with correct CCs, SC-006 shortage of 6, SC-008 overage of 8, and variance attachment.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to daniel.reyes@prairiestar.com for manifest 2026-04-15-002 with CCs to carlos.mendoza and jacob.patel, mentioning SC-006 short 6 cases, SC-008 extra 8 cases, and attachment IR-001-2026-04-15-Variance.docx. Check sent emails in mailbox.json.\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "9ea00608-f5de-4b40-8e6f-e77876073e76",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), a sent email to jacob.patel@prairiestar.com must exist that CCs emily.carter@prairiestar.com, contains the text 'Inbound short-dated product', references SKU 'SW-008', lot code 'C-20260201-01', days to expiry '15' or '16', and requests prioritization in outbound.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n for email in emails:\n if email.get(\"folder\", \"\").lower() not in [\"sent\"]:\n continue\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n if \"jacob.patel@prairiestar.com\" not in to_addr:\n continue\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n if \"emily.carter@prairiestar.com\" not in cc_addr:\n continue\n body = (email.get(\"body_text\") or \"\")\n subject = (email.get(\"subject\") or \"\")\n combined = body + \" \" + subject\n combined_lower = combined.lower()\n if \"inbound short-dated product\" not in combined_lower:\n continue\n if \"sw-008\" not in combined_lower:\n continue\n if \"c-20260201-01\" not in combined_lower:\n continue\n # Accept either 15 or 16 days to expiry\n has_days = (\"15\" in combined or \"16\" in combined)\n if not has_days:\n continue\n if not any(w in combined_lower for w in [\"priorit\", \"outbound\"]):\n continue\n days_found = []\n if \"15\" in combined:\n days_found.append(\"15\")\n if \"16\" in combined:\n days_found.append(\"16\")\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email to jacob.patel CC emily.carter with short-dated product alert for SW-008, lot C-20260201-01, {'/'.join(days_found)} days to expiry, outbound prioritization request\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to jacob.patel@prairiestar.com CC emily.carter with required fields: 'Inbound short-dated product', SW-008, C-20260201-01, 15 or 16 days, outbound prioritization\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "1ae74eeb-a8a8-4e8e-b33b-a1ac03866295",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), a sent email to daniel.reyes@prairiestar.com must exist that references SKU 'SW-008', lot code 'C-20260201-01', days to expiry '15' or '16', and requests an immediate demand plan.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n candidates = []\n for email in emails:\n folder = (email.get(\"folder\") or \"\").lower()\n if folder != \"sent\":\n continue\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n if \"daniel.reyes@prairiestar.com\" not in to_addr:\n continue\n body = email.get(\"body_text\") or \"\"\n subject = email.get(\"subject\") or \"\"\n combined = body + \" \" + subject\n combined_lower = combined.lower()\n candidates.append(combined_lower)\n # Check SKU SW-008\n has_sku = \"sw-008\" in combined_lower or \"sw008\" in combined_lower\n if not has_sku:\n continue\n # Check lot code C-20260201-01\n has_lot = \"c-20260201-01\" in combined_lower or \"c20260201-01\" in combined_lower or \"c-2026020101\" in combined_lower\n if not has_lot:\n continue\n # Check days to expiry: 15 or 16\n has_expiry = any(d in combined for d in [\"15\", \"16\"])\n if not has_expiry:\n continue\n # Check demand plan request\n has_demand = \"demand plan\" in combined_lower or (\"demand\" in combined_lower and \"plan\" in combined_lower)\n if not has_demand:\n continue\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found sent email to daniel.reyes@prairiestar.com referencing SW-008, lot C-20260201-01, days to expiry 15 or 16, and demand plan request.\"}\n if candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(candidates)} sent email(s) to daniel.reyes@prairiestar.com but none contained all required elements: SKU 'SW-008', lot 'C-20260201-01', days to expiry '15' or '16', and demand plan request. Email contents (lowered): {candidates[:3]}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found addressed to daniel.reyes@prairiestar.com. Checked all emails in mailbox.json sent folder.\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "a1d0e6f6-6d92-4a7f-a71d-fcd510a72d6b",
|
|||
|
|
"sort_order": 4,
|
|||
|
|
"rubric_text": "In `slack_data.json` (external services), the '#dc-escalations' channel must contain a post referencing manifest '2026-04-15-003', the text 'P2', variance, the text '@d.reyes' or 'escalated'.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n # Find dc-escalations channel\n dc_channel_id = None\n for ch_id, ch in channels.items():\n if \"dc-escalations\" in ch.get(\"name\", \"\").lower():\n dc_channel_id = ch_id\n break\n if dc_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #dc-escalations channel in slack_data.json\"}\n messages = data.get(\"messages\", {})\n channel_msgs = messages.get(dc_channel_id, [])\n\n best_match_details = []\n\n for msg in channel_msgs:\n text = (msg.get(\"text\") or \"\").lower()\n missing = []\n\n # Must reference manifest 2026-04-15-003 (be loose: accept 003, 2026-04-15-003, manifest 003, etc.)\n if \"2026-04-15-003\" not in text and \"003\" not in text:\n missing.append(\"manifest '2026-04-15-003'\")\n\n # Must contain P2 (be loose: p2, P2, priority 2, etc.)\n if not any(kw in text for kw in [\"p2\", \"priority 2\", \"priority: 2\", \"priority-2\"]):\n missing.append(\"'P2'\")\n\n # Must reference variance (be very loose: variance, discrepancy, deviation, temperature issue, damage, mismatch, off, etc.)\n if not any(kw in text for kw in [\"variance\", \"discrepancy\", \"deviation\", \"temperature\", \"damage\", \"issue\", \"mismatch\", \"off\", \"incorrect\", \"wrong\", \"differ\", \"anomal\", \"exceed\", \"outside\", \"range\", \"threshold\", \"alert\", \"flag\"]):\n missing.append(\"variance\")\n\n # Must contain @d.reyes or 'escalated' or similar escalation language\n if not any(kw in text for kw in [\"d.reyes\", \"reyes\", \"escalat\"]):\n missing.append(\"'@d.reyes' or 'escalated'\")\n\n if not missing:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found dc-escalations post for manifest 2026-04-15-003 with P2, variance, and escalation to d.reyes. Message: {msg.get('text', '')[:200]}\"}\n\n # Track the best near-match for feedback\n if len(missing) <= 2:\n best_match_details.append({\"text\": msg.get(\"text\", \"\")[:200], \"missing\": missing})\n\n if best_match_details:\n closest = min(best_match_details, key=lambda x: len(x[\"missing\"]))\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found near-match in #dc-escalations but missing: {', '.join(closest['missing'])}. Closest message: {closest['text']}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No post found in #dc-escalations with required fields for manifest 2026-04-15-003 (P2, variance, @d.reyes/escalated). Found {len(channel_msgs)} messages in channel.\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "5f66b70c-8491-474f-8cb5-20b57a39543a",
|
|||
|
|
"sort_order": 5,
|
|||
|
|
"rubric_text": "In `slack_data.json` (external services), the '#waco-production-coord' channel must contain a post referencing manifest '2026-04-15-002', SKUs 'SC-006' and 'SC-008', the order being short 6 cases of SC-006, having 8 extra cases of SC-008, variance tier 'Major', and 'Email sent. Status: Open'.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n waco_channel_id = None\n for ch_id, ch in channels.items():\n if \"waco-production-coord\" in ch.get(\"name\", \"\").lower():\n waco_channel_id = ch_id\n break\n if waco_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #waco-production-coord channel in slack_data.json\"}\n messages = data.get(\"messages\", {})\n channel_msgs = messages.get(waco_channel_id, [])\n for msg in channel_msgs:\n text = (msg.get(\"text\") or \"\").lower()\n if \"2026-04-15-002\" not in text:\n continue\n if \"sc-006\" not in text:\n continue\n if \"sc-008\" not in text:\n continue\n if \"6\" not in text:\n continue\n if \"8\" not in text:\n continue\n if \"major\" not in text:\n continue\n if \"open\" not in text:\n continue\n if \"email\" not in text:\n continue\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found waco-production-coord post for manifest 2026-04-15-002 with SC-006, SC-008, 6 short, 8 extra, Major variance, Email sent, Status Open\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No post found in #waco-production-coord with all required fields for manifest 2026-04-15-002 (SC-006, SC-008, short 6, extra 8, Major, Email sent, Status Open)\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "05e2327c-3645-4b6b-9308-590af59ee3b2",
|
|||
|
|
"sort_order": 6,
|
|||
|
|
"rubric_text": "In `slack_data.json` (external services), the '#dc-escalations' channel must contain a post referencing manifest '2026-04-15-002', the text 'major', 'P2', variance, the text '@d.reyes' or 'escalated' with status 'Open'.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n dc_channel_id = None\n for ch_id, ch in channels.items():\n if \"dc-escalations\" in ch.get(\"name\", \"\").lower():\n dc_channel_id = ch_id\n break\n if dc_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #dc-escalations channel\"}\n messages = data.get(\"messages\", {})\n channel_msgs = messages.get(dc_channel_id, [])\n for msg in channel_msgs:\n text = (msg.get(\"text\") or \"\").lower()\n # Must reference manifest 2026-04-15-002\n if \"2026-04-15-002\" not in text and \"002\" not in text:\n continue\n # Must reference 'major'\n if \"major\" not in text:\n continue\n # Must reference P2\n if \"p2\" not in text:\n continue\n # Must reference variance (loose: accept \"variance\", \"discrepancy\", \"varian\")\n if not any(w in text for w in [\"variance\", \"discrepan\", \"varian\"]):\n continue\n # Must reference @d.reyes or escalated\n if not any(w in text for w in [\"d.reyes\", \"reyes\", \"escalat\"]):\n continue\n # Must have status Open (loosely matched)\n if \"open\" not in text:\n continue\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found dc-escalations post for manifest 2026-04-15-002 with 'major', P2, variance, escalation reference, and Open status.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No post found in #dc-escalations with all required fields for manifest 2026-04-15-002 (major, P2, variance, @d.reyes or escalated, Open). Found \" + str(len(channel_msgs)) + \" messages in channel.\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "45a75041-0c87-4b73-9219-91f47f1e25f6",
|
|||
|
|
"sort_order": 7,
|
|||
|
|
"rubric_text": "In `inventory_master.xlsx`, the file must remain unchanged and contain exactly 28 rows of data (not counting the header).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob(\"**/inventory_master.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n xlsx_files = [f for f in xlsx_files if \"inventory_master\" in f.name.lower()]\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"inventory_master.xlsx not found in workspace\"}\n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n # Count non-empty rows excluding header\n data_rows = 0\n for row in ws.iter_rows(min_row=2):\n if any(cell.value is not None for cell in row):\n data_rows += 1\n if data_rows == 28:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"inventory_master.xlsx contains exactly 28 data rows as expected (file unchanged).\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"inventory_master.xlsx contains {data_rows} data rows, expected exactly 28. The file must remain unchanged.\"}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "0911ee42-1e48-472e-a072-07c2b718b70c",
|
|||
|
|
"sort_order": 8,
|
|||
|
|
"rubric_text": "In `expiry_tracker.xlsx`, the file must remain unchanged and contain exactly 28 rows of data (not counting the header).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Find expiry_tracker.xlsx\n xlsx_files = list(Path(workspace_path).glob(\"**/expiry_tracker.xlsx\"))\n if not xlsx_files:\n xlsx_files = [f for f in Path(workspace_path).glob(\"**/*.xlsx\") if \"expiry_tracker\" in f.name.lower()]\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"expiry_tracker.xlsx not found in workspace\"}\n \n wb = openpyxl.load_workbook(xlsx_files[0])\n ws = wb.active\n \n # Count data rows (non-empty rows excluding the header)\n # The header is assumed to be row 1\n data_rows = 0\n for row in ws.iter_rows(min_row=2):\n # Check if the row has any non-None value\n if any(cell.value is not None for cell in row):\n data_rows += 1\n \n if data_rows == 28:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"expiry_tracker.xlsx contains exactly 28 data rows (not counting the header), as expected. The file appears unchanged.\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"expiry_tracker.xlsx must contain exactly 28 rows of data (not counting the header), but found {data_rows} data rows. The file should remain unchanged.\"\n }\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776307678542",
|
|||
|
|
"sort_order": 9,
|
|||
|
|
"rubric_text": "File `temperature_log.xlsx` should satisfy all of the following:\n\n1. Cells A44–A52 should contain the value \"2026-04-15\".\n2. Cells B44–B46 should contain the value \"09:00\" or \"09:15\".\n3. Cells B47–B49 should contain the value \"10:00\" or \"10:15\".\n4. Cells B50–B52 should contain the value \"11:00\" or \"11:15\".\n5. Cells C44–C46 should contain the value \"2026-04-15-001\".\n6. Cells C47–C49 should contain the value \"2026-04-15-002\".\n7. Cells C50–C52 should contain the value \"2026-04-15-003\".\n8. Cell D44 should contain \"-18\", D45 should contain \"-16\", D46 should contain \"-20\".\n9. Cell D47 should contain \"-18\", D48 should contain \"-16\", D49 should contain \"-20\".\n10. Cell D50 should contain \"-5\", D51 should contain \"-4\", D52 should contain \"-4\".\n11. Cells E44, E47, and E50 should contain 'Inbound' and 'Front'.\n12. Cells E45, E48, and E51 should contain 'Inbound' and 'Middle'.\n13. Cells E46, E49, and E52 should contain 'Inbound' and 'Rear'.\n14. Cells F44–F46 should contain \"Y\".\n15. Cells F47–F49 should contain \"Y\".\n16. Cells F50–F52 should contain \"N\".\n17. Cells H44–H52 should contain \"A. Nguyen\".",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\n\ndef _cell_str(val):\n \"\"\"Normalize a cell value to a comparable string.\"\"\"\n if val is None:\n return \"\"\n return str(val).strip()\n\n\ndef _num_close(cell_val, expected, tol=1.0):\n \"\"\"Check whether a cell value is numerically close to expected (absolute tolerance).\"\"\"\n try:\n s = str(cell_val).strip()\n # Remove degree symbols, C/F suffixes\n s = re.sub(r'[°CF]+', '', s).strip()\n cv = float(s)\n ev = float(expected)\n return abs(cv - ev) <= tol\n except (ValueError, TypeError):\n return False\n\n\ndef _text_contains(cell_val, substring):\n cv = _cell_str(cell_val).lower()\n return substring.lower() in cv\n\n\ndef _date_match(cell_val, expected_str):\n \"\"\"Accept '2026-04-15' as string, or a datetime date.\"\"\"\n cv = cell_val\n if cv is None:\n return False\n if isinstance(cv, datetime.datetime):\n return cv.strftime(\"%Y-%m-%d\") == expected_str\n if isinstance(cv, datetime.date):\n return cv.strftime(\"%Y-%m-%d\") == expected_str\n s = str(cv).strip()\n if s == expected_str:\n return True\n # Try parsing\n for fmt in (\"%Y-%m-%d\", \"%m/%d/%Y\", \"%Y/%m/%d\", \"%d-%m-%Y\"):\n try:\n d = datetime.datetime.strptime(s, fmt)\n if d.strftime(\"%Y-%m-%d\") == expected_str:\n return True\n except ValueError:\n pass\n return False\n\n\ndef _time_match_any(cell_val, expected_options):\n \"\"\"Check if cell_val matches any of the expected time strings.\"\"\"\n cv = cell_val\n if cv is None:\n return False\n for expected_str in expected_options:\n if _time_match_single(cv, expected_str):\n return True\n return False\n\n\ndef _time_match_single(cell_val, expected_str):\n \"\"\"Accept '09:00', '9:00', datetime.time, etc.\"\"\"\n cv = cell_val\n if cv is None:\n return False\n if isinstance(cv, datetime.time):\n parts = expected_str.split(\":\")\n return cv.hour == int(parts[0]) and cv.minute == int(parts[1])\n if isinstance(cv, datetime.datetime):\n parts = expected_str.split(\":\")\n return cv.hour == int(parts[0]) and cv.minute == int(parts[1])\n s = str(cv).strip()\n def norm_time(t):\n p = t.split(\":\")\n if len(p) >= 2:\n return f\"{int(p[0])}:{p[1].zfill(2)}\"\n return t\n return norm_time(s) == norm_time(expected_str) or s == expected_str\n\n\ndef _find_file(workspace_path):\n \"\"\"Find temperature_log.xlsx, being flexible about location.\"\"\"\n fp = Path(workspace_path) / \"temperature_log.xlsx\"\n if fp.exists():\n return fp\n results = list(Path(workspace_path).glob(\"**/temperature_log.xlsx\"))\n if results:\n return results[0]\n results = list(Path(workspace_path).glob(\"**/*temperature*.xlsx\"))\n if results:\n return results[0]\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n fp = _find_file(workspace_path)\n if fp is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File temperature_log.xlsx not found.\"}\n\n wb = openpyxl.load_workbook(fp, data_only=True)\n ws = wb.active\n\n total_checks = 0\n passed_checks = 0\n issues = []\n\n def check(ok, desc):\n nonlocal total_checks, passed_checks\n total_checks += 1\n if ok:\n passed_checks += 1\n else:\n issues.append(desc)\n\n # 1. A44-A52 = 2026-04-15\n for r in range(44, 53):\n val = ws.cell(row=r, column=1).value\n check(_date_match(val, \"2026-04-15\"), f\"A{r}: expected date '2026-04-15', got '{val}'\")\n\n # 2. B44-B46 = 09:00 or 09:15\n for r in range(44, 47):\n val = ws.cell(row=r, column=2).value\n check(_time_match_any(val, [\"09:00\", \"09:15\"]), f\"B{r}: expected '09:00' or '09:15', got '{val}'\")\n\n # 3
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776308591925",
|
|||
|
|
"sort_order": 10,
|
|||
|
|
"rubric_text": "The file `inbound_receiving_log.xlsx` should contain the following values in row 12: \"2026-04-15\" in column A, \"2026-04-15-001\" in column B, \"Waco Transfer\" in column C, \"26\" in column D, \"26\" in column E, \"492\" in column F, \"492\" in column G, \"-18\" in column H, \"-16\" in column I, \"-20\" in column J, \"-18\" in column K, \"Tier 1\" in column L, ~0 in column M, ~0 or 0% in column N, and \"Pending Put-Away\" in column P or column Q.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find inbound_receiving_log.xlsx - search broadly\n filepath = Path(workspace_path) / 'inbound_receiving_log.xlsx'\n if not filepath.exists():\n # Try glob\n candidates = list(Path(workspace_path).glob('**/inbound_receiving_log.xlsx'))\n if candidates:\n filepath = candidates[0]\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: inbound_receiving_log.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(filepath))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening workbook: {e}\"}\n\n # Try to find the correct row - check rows 2 through 30 for our shipment ID\n target_row = None\n for r in range(2, 31):\n val_b = ws[f\"B{r}\"].value\n if val_b is not None:\n val_str = str(val_b).strip()\n if '2026-04-15-001' in val_str:\n target_row = r\n break\n \n # If not found by shipment ID, try finding by date in column A\n if target_row is None:\n for r in range(2, 31):\n val_a = ws[f\"A{r}\"].value\n if val_a is not None:\n if isinstance(val_a, (datetime, date)):\n if val_a.strftime(\"%Y-%m-%d\") == \"2026-04-15\":\n target_row = r\n break\n elif '2026-04-15' in str(val_a):\n target_row = r\n break\n \n # Default to row 12 as specified in rubric\n if target_row is None:\n target_row = 12\n\n row = target_row\n issues = []\n passes = []\n total_checks = 15 # A, B, C, D, E, F, G, H, I, J, K, L, M, N, and P-or-Q status\n\n def cell_val(col_letter):\n cell = ws[f\"{col_letter}{row}\"]\n return cell.value\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, (datetime, date)):\n return val.strftime(\"%Y-%m-%d\")\n return str(val).strip()\n\n def check_numeric(val, expected, label, tolerance=0.1):\n nval = normalize(val)\n # Try to extract a number from the string\n try:\n num_val = float(nval)\n except (ValueError, TypeError):\n # Try regex to find a number\n match = re.search(r'-?[\\d.]+', nval)\n if match:\n try:\n num_val = float(match.group())\n except (ValueError, TypeError):\n return False, f\"{label}: expected ~{expected}, got '{nval}' (not numeric)\"\n else:\n return False, f\"{label}: expected ~{expected}, got '{nval}' (not numeric)\"\n try:\n num_exp = float(expected)\n except (ValueError, TypeError):\n return False, f\"{label}: could not parse expected value '{expected}'\"\n if num_exp == 0:\n if abs(num_val) < 1:\n return True, f\"{label}: found {num_val} (expected {num_exp})\"\n else:\n return False, f\"{label}: expected {num_exp}, got {num_val}\"\n if abs(num_val - num_exp) / max(abs(num_exp), 1e-9) <= tolerance:\n return True, f\"{label}: found {num_val} (expected {num_exp})\"\n return False, f\"{label}: expected {num_exp}, got {num_val}\"\n\n def check_string(val, expected, label, exact=False):\n nval = normalize(val).lower()\n nexp = expected.lower().strip()\n if exact:\n if nval == nexp:\n return True, f\"{label}: found '{normalize(val)}'\"\n return False, f\"{label}: expected '{expected}', got '{normalize(val)}'\"\n if nexp in nval or nval == nexp:\n return True, f\"{label}: found '{normalize(val)}'\"\n return False
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776309041156",
|
|||
|
|
"sort_order": 11,
|
|||
|
|
"rubric_text": "The file `inbound_receiving_log.xlsx` should contain, in row 14, the following values: \"2026-04-15\" in column A, \"2026-04-15-003\" in column B, \"Waco Transfer\" in column C, \"26\" in column D, \"26\" in column E, \"520\" in column F, \"516\" in column G, \"-5\" in column H, \"-4\" in column I, \"-4\" in column J, \"-4\" in column K, \"Tier 3\" in column L, \"-4\" in column M, \"3%\" in column N, and the text \"Received - Pending Put-Away\" in column P or column Q.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file using glob first, then exact path\n workspace = Path(workspace_path)\n candidates = list(workspace.glob('**/inbound_receiving_log.xlsx'))\n if candidates:\n file_path = candidates[0]\n else:\n file_path = workspace / 'inbound_receiving_log.xlsx'\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: inbound_receiving_log.xlsx in {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n\n # We'll search rows 10-25 for a row that contains '2026-04-15-003' in column B\n target_row = None\n for r in range(10, 25):\n cell_val = ws.cell(row=r, column=2).value\n if cell_val is not None and '003' in str(cell_val):\n target_row = r\n break\n if target_row is None:\n target_row = 14 # fallback to expected row\n\n row = target_row\n errors = []\n matches = []\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%Y-%m-%d\")\n if isinstance(val, date):\n return val.strftime(\"%Y-%m-%d\")\n return str(val).strip()\n\n def numbers_close(actual_str, expected_str, tol=0.15):\n \"\"\"Check if two values are numerically close within tolerance.\"\"\"\n try:\n a_str = actual_str.replace('%', '').replace(',', '').replace('\\u2212', '-').replace('\\u2013', '-').strip()\n e_str = expected_str.replace('%', '').replace(',', '').strip()\n a = float(a_str)\n e = float(e_str)\n if e == 0:\n return abs(a) < 1.0\n return abs(a - e) / max(abs(e), 1e-9) <= tol\n except (ValueError, AttributeError):\n return False\n\n def check_cell(col_letter, expected, col_idx):\n cell_val = ws.cell(row=row, column=col_idx).value\n actual = normalize(cell_val)\n # Direct string match (case-insensitive)\n if actual.lower() == expected.lower():\n matches.append(f\"Column {col_letter} (row {row}): found '{actual}' == '{expected}'\")\n return True\n # Check if expected is contained in actual\n if expected.lower() in actual.lower():\n matches.append(f\"Column {col_letter} (row {row}): found '{actual}' contains '{expected}'\")\n return True\n # Numeric close match\n if numbers_close(actual, expected):\n matches.append(f\"Column {col_letter} (row {row}): found '{actual}' ~= '{expected}' (numeric match)\")\n return True\n # For percentage: handle '3%' vs '0.03' or '3.0%' etc.\n if '%' in expected:\n exp_num = expected.replace('%', '').strip()\n if cell_val is not None:\n try:\n fval = float(cell_val)\n # openpyxl might store 3% as 0.03\n if abs(fval - float(exp_num) / 100.0) < 0.015:\n matches.append(f\"Column {col_letter} (row {row}): found '{cell_val}' matches '{expected}' as decimal\")\n return True\n # Or it might store as 3.0 meaning 3%\n if abs(fval - float(exp_num)) < 1.5:\n matches.append(f\"Column {col_letter} (row {row}): found '{cell_val}' matches '{expected}'\")\n return True\n except (ValueError, TypeError):\n pass\n # Also check if actual string contains %\n if '%' in actual:\n try:\n actual_pct = float(actual.replace('%', '').strip())\n
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776309295974",
|
|||
|
|
"sort_order": 12,
|
|||
|
|
"rubric_text": "The file `inbound_receiving_log.xlsx` should contain, in row 13, the following values: \"2026-04-15\" in column A, \"2026-04-15-002\" in column B, \"Waco Transfer\" in column C, \"26\" in column D, \"26\" in column E, \"520\" in column F, \"522\" in column G, \"-18\" in column H, \"-16\" in column I, \"-20\" in column J, \"-18\" in column K, \"Tier 1\" in column L, a non-zero value in column M, a non-zero value in column N, and the text \"Received - Pending Put-Away\" in column P or column Q.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n filepath = Path(workspace_path) / 'inbound_receiving_log.xlsx'\n if not filepath.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {filepath}\"}\n \n wb = openpyxl.load_workbook(str(filepath))\n ws = wb.active\n \n row = 13\n feedback_parts = []\n checks_passed = 0\n total_checks = 0\n \n def cell_val(r, c):\n v = ws.cell(row=r, column=c).value\n return v\n \n def normalize(v):\n if v is None:\n return ''\n s = str(v).strip()\n return s\n \n def check_numeric(actual, expected, label, tolerance=0.05):\n nonlocal checks_passed, total_checks\n total_checks += 1\n actual_str = normalize(actual)\n try:\n actual_num = float(actual_str)\n expected_num = float(expected)\n if expected_num == 0:\n if abs(actual_num) < 0.5:\n checks_passed += 1\n feedback_parts.append(f\"PASS {label}: found {actual_str}, expected {expected}\")\n return True\n else:\n feedback_parts.append(f\"FAIL {label}: found {actual_str}, expected {expected}\")\n return False\n if abs(actual_num - expected_num) / max(abs(expected_num), 1e-9) <= tolerance:\n checks_passed += 1\n feedback_parts.append(f\"PASS {label}: found {actual_str}, expected {expected}\")\n return True\n # Also try exact match\n if actual_num == expected_num:\n checks_passed += 1\n feedback_parts.append(f\"PASS {label}: found {actual_str}, expected {expected}\")\n return True\n feedback_parts.append(f\"FAIL {label}: found {actual_str}, expected {expected}\")\n return False\n except (ValueError, TypeError):\n # Try string match\n if actual_str == str(expected):\n checks_passed += 1\n feedback_parts.append(f\"PASS {label}: found {actual_str} (string match)\")\n return True\n feedback_parts.append(f\"FAIL {label}: found '{actual_str}', expected {expected}\")\n return False\n \n def check_string(actual, expected, label, loose=False):\n nonlocal checks_passed, total_checks\n total_checks += 1\n actual_str = normalize(actual)\n expected_str = str(expected).strip()\n if loose:\n if expected_str.lower() in actual_str.lower():\n checks_passed += 1\n feedback_parts.append(f\"PASS {label}: found '{actual_str}' contains '{expected_str}'\")\n return True\n else:\n if actual_str.lower() == expected_str.lower():\n checks_passed += 1\n feedback_parts.append(f\"PASS {label}: found '{actual_str}', expected '{expected_str}'\")\n return True\n feedback_parts.append(f\"FAIL {label}: found '{actual_str}', expected '{expected_str}'\")\n return False\n \n def check_date(actual, expected_date_str, label):\n nonlocal checks_passed, total_checks\n total_checks += 1\n actual_str = normalize(actual)\n # Handle datetime objects\n import datetime\n if isinstance(actual, (datetime.datetime, datetime.date)):\n formatted = actual.strftime('%Y-%m-%d')\n if formatted == expected_date_str or actual_str.startswith(expected_date_str):\n checks_passed += 1\n feedback_parts.append(f\"PASS {label}: found date {formatted}, expected {expected_date_str}\")\n return True\n if expected_date_str in actual_str:\n checks_passed += 1\n feedback_parts.append(f\"PASS {label}: found '{actual_str}' contains '{expected_date_str}'\")\n r
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776309397046",
|
|||
|
|
"sort_order": 13,
|
|||
|
|
"rubric_text": "The docx file containing the text `IR-001-2026-04-15' in its title should exist in the workspace and contain the following:\n- Date: 2026-04-15\n- A location mentioning Houston\n- Manifest number: 2026-04-15-002\n- Priority level: P2\n- A mention of variation\n- Actions taken including holding the order open and mentions of escalating to Daniel Reyes and Carlos Mendoza via email\n- Mention of notifying Daniel Reyes\n- Attachments including quantity variance details of SC-006 and SC-008",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Find docx file with IR-001-2026-04-15 in the title\n all_docx = list(Path(workspace_path).rglob(\"*.docx\"))\n matches = [f for f in all_docx if \"IR-001\" in f.name and \"2026-04-15\" in f.name]\n if not matches:\n matches = [f for f in all_docx if \"IR-001\" in f.name]\n if not matches:\n matches = [f for f in all_docx if \"IR\" in f.name and \"001\" in f.name]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No docx file with 'IR-001-2026-04-15' in title found. Found docx files: {[str(f.name) for f in all_docx]}\"}\n \n file_path = matches[0]\n \n # Read content from docx\n try:\n from docx import Document\n doc = Document(str(file_path))\n paragraphs_text = [p.text for p in doc.paragraphs]\n \n # Also get text from tables\n table_texts = []\n for table in doc.tables:\n for row in table.rows:\n for cell in row.cells:\n table_texts.append(cell.text)\n \n full_text = \"\\n\".join(paragraphs_text + table_texts).lower()\n full_text_original = \"\\n\".join(paragraphs_text + table_texts)\n except ImportError:\n import zipfile\n try:\n with zipfile.ZipFile(str(file_path), 'r') as z:\n with z.open('word/document.xml') as f:\n raw = f.read().decode('utf-8', errors='replace')\n full_text = re.sub(r'<[^>]+>', ' ', raw).lower()\n full_text_original = re.sub(r'<[^>]+>', ' ', raw)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read file: {e}\"}\n \n total = 0\n passed = 0\n feedback_parts = []\n \n # 1. Date 2026-04-15\n total += 1\n date_patterns = [\"2026-04-15\", \"04/15/2026\", \"april 15, 2026\", \"april 15 2026\", \"04-15-2026\", \"15 april 2026\", \"4/15/2026\", \"4-15-2026\"]\n date_found = any(p in full_text for p in date_patterns)\n if date_found:\n passed += 1\n feedback_parts.append(\"PASS: Date 2026-04-15 found.\")\n else:\n feedback_parts.append(\"FAIL: Date 2026-04-15 not found in document.\")\n \n # 2. Location mentioning Houston\n total += 1\n houston_found = \"houston\" in full_text\n if houston_found:\n passed += 1\n feedback_parts.append(\"PASS: Houston location found.\")\n else:\n feedback_parts.append(\"FAIL: Houston location not found in document.\")\n \n # 3. Manifest number 2026-04-15-002\n total += 1\n manifest_patterns = [\"2026-04-15-002\", \"2026_04_15_002\", \"2026/04/15/002\"]\n manifest_found = any(p in full_text for p in manifest_patterns)\n if manifest_found:\n passed += 1\n feedback_parts.append(\"PASS: Manifest number 2026-04-15-002 found.\")\n else:\n feedback_parts.append(\"FAIL: Manifest number 2026-04-15-002 not found.\")\n \n # 4. Priority level P2\n total += 1\n p2_found = re.search(r'p\\s*[-:]?\\s*2|priority\\s*[-:]?\\s*2|p2', full_text) is not None\n if p2_found:\n passed += 1\n feedback_parts.append(\"PASS: Priority level P2 found.\")\n else:\n feedback_parts.append(\"FAIL: Priority level P2 not found.\")\n \n # 5. A mention of variation (variance, variation, discrepancy, etc.)\n total += 1\n variation_found = any(w in full_text for w in [\"variance\", \"variation\", \"discrepancy\", \"discrepancies\"])\n if variation_found:\n passed += 1\n feedback_parts.append(\"PASS: Variation/variance mentioned.\")\n else:\n feedback_parts.append(\"FAIL: No mention of variation/variance/discrepancy found.\")\n \n # 6. Holding the order open\n total += 1\n hold_found = any(p in full_text for p in [\"hold\", \"held\", \"holding\", \"open\"])\n if hold_found:\n passed += 1\n feedback_parts.append(\"PASS:
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776309609324",
|
|||
|
|
"sort_order": 14,
|
|||
|
|
"rubric_text": "The docx file containing the text `IR-002-2026-04-15' in its title should contain: the date 2026-04-15, a location mentioning Houston, manifest number 2026-04-15-003, priority level P2, a mention of variation, actions taken including quarantining the order and mentions of escalating to Daniel Reyes and Carlos Mendoza via email, mention of notifying Daniel Reyes, and attachments including temperature readings (-5, -4, and -4.5) and quantity variance (expected 520, got 516).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Find any docx file whose name contains 'IR-002-2026-04-15' (loose matching)\n workspace = Path(workspace_path)\n candidates = list(workspace.rglob(\"*IR-002*2026*04*15*.docx\")) + list(workspace.rglob(\"*IR*002*2026*04*15*.docx\"))\n # Deduplicate\n seen = set()\n unique_candidates = []\n for c in candidates:\n if c not in seen:\n seen.add(c)\n unique_candidates.append(c)\n candidates = unique_candidates\n\n if not candidates:\n # Also try broader search for any docx with IR-002 in the name\n candidates = list(workspace.rglob(\"*IR*002*.docx\"))\n\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No docx file containing 'IR-002-2026-04-15' in its title was found in the workspace.\"}\n\n file_path = candidates[0]\n\n try:\n from docx import Document\n doc = Document(str(file_path))\n full_text = \"\\n\".join([p.text for p in doc.paragraphs])\n # Also check tables\n for table in doc.tables:\n for row in table.rows:\n for cell in row.cells:\n full_text += \"\\n\" + cell.text\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read '{file_path.name}': {e}\"}\n\n text_lower = full_text.lower()\n text_norm = re.sub(r'\\s+', ' ', full_text)\n text_norm_lower = text_norm.lower()\n\n total = 0\n passed = 0\n feedback_parts = []\n\n def check(name, condition, detail=\"\"):\n nonlocal total, passed\n total += 1\n if condition:\n passed += 1\n feedback_parts.append(f\"PASS: {name} {detail}\")\n else:\n feedback_parts.append(f\"FAIL: {name} {detail}\")\n return condition\n\n # 1. Date 2026-04-15\n date_found = any(d in full_text for d in [\"2026-04-15\", \"04/15/2026\", \"April 15, 2026\", \"04-15-2026\", \"2026/04/15\", \"15 April 2026\", \"April 15 2026\", \"4/15/2026\", \"15/04/2026\"])\n check(\"Date 2026-04-15\", date_found)\n\n # 2. Location mentions Houston\n check(\"Location mentions Houston\", \"houston\" in text_lower)\n\n # 3. Manifest number 2026-04-15-003\n manifest_found = bool(re.search(r'2026[\\-/\\.]?04[\\-/\\.]?15[\\-/\\.]?003', full_text))\n check(\"Manifest number 2026-04-15-003\", manifest_found)\n\n # 4. Priority level P2\n p2_found = bool(re.search(r'P[\\s\\-]?2|priority[\\s:]*2', full_text, re.IGNORECASE))\n check(\"Priority level P2\", p2_found)\n\n # 5. Mention of variation (temperature or quantity or general)\n variation_found = any(t in text_lower for t in [\"varia\", \"discrepan\", \"deviation\", \"excursion\", \"anomal\"])\n check(\"Mention of variation\", variation_found)\n\n # 6. Quarantine action\n quarantine_found = \"quarantin\" in text_lower\n check(\"Quarantine action mentioned\", quarantine_found)\n\n # 7. Escalating to Daniel Reyes via email\n daniel_found = any(d in text_lower for d in [\"daniel reyes\", \"d. reyes\", \"dreyes\", \"daniel r.\", \"d.reyes\", \"daniel_reyes\"])\n email_mention = any(e in text_lower for e in [\"email\", \"e-mail\", \"escalat\", \"notif\"])\n check(\"Escalation to Daniel Reyes (email)\", daniel_found and email_mention, f\"(daniel:{daniel_found}, email:{email_mention})\")\n\n # 8. Escalating to Carlos Mendoza via email\n carlos_found = any(c in text_lower for c in [\"carlos mendoza\", \"c. mendoza\", \"cmendoza\", \"carlos m.\", \"c.mendoza\", \"carlos_mendoza\"])\n check(\"Escalation to Carlos Mendoza (email)\", carlos_found and email_mention, f\"(carlos:{carlos_found}, email:{email_mention})\")\n\n # 9. Notifying Daniel Reyes (general mention - rubric says \"mention of notifying Daniel Reyes\")\n notify_daniel = daniel_found and any(n in text_lower for n in [\"notif\", \"alert\", \"inform\", \"contact\", \"escalat\", \"communicat\", \"
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
}
|
|||
|
|
]
|