122 lines
55 KiB
JSON
122 lines
55 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "7fa2795d-558f-47a3-a6ab-4041da8f28eb",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In file `ncr_log.xlsx`, a new row must exist with NCR_Number=NCR-MER-2026-0032, Issue_Date=03/12/2026, Supplier_ID=SUP-0010, Supplier_Name=Redstone Alloy Fabricators Inc, PO_Number=MER-PO-103517, Receipt_ID=REC-20260312-001, SKU=FL-20044-A3, Quantity_Affected=80, NCR_Type=DOC, Severity=Major, Dollar_Value=$3780.00, Status=Open, Issued_By=DH, Supplier_Response_Due=03/26/2026, and an empty Description.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date as date_type\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n ncr_files = [f for f in xlsx_files if 'ncr_log' in f.name.lower()]\n if not ncr_files:\n ncr_files = [f for f in xlsx_files if 'ncr' in f.name.lower()]\n if not ncr_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'ncr_log.xlsx not found in workspace'}\n\n ncr_file = ncr_files[0]\n try:\n wb = openpyxl.load_workbook(ncr_file, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open {ncr_file.name}: {e}'}\n\n headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[1]]\n\n def col_idx(name):\n name_norm = re.sub(r'[^a-z0-9]', '', name.lower())\n for i, h in enumerate(headers):\n if re.sub(r'[^a-z0-9]', '', h.lower()) == name_norm:\n return i\n return None\n\n def norm_str(val):\n if val is None:\n return ''\n return re.sub(r'[^a-z0-9]', '', str(val).lower())\n\n def norm_date(val):\n if val is None:\n return ''\n if isinstance(val, datetime):\n return val.strftime('%Y%m%d')\n if isinstance(val, date_type):\n return val.strftime('%Y%m%d')\n s = str(val).strip()\n m = re.match(r'^(\\d{1,2})[/\\-](\\d{1,2})[/\\-](\\d{4})$', s)\n if m:\n return f'{m.group(3)}{int(m.group(1)):02d}{int(m.group(2)):02d}'\n m = re.match(r'^(\\d{4})[/\\-](\\d{2})[/\\-](\\d{2})', s)\n if m:\n return f'{m.group(1)}{m.group(2)}{m.group(3)}'\n return ''\n\n def norm_num(val):\n if val is None:\n return None\n try:\n return float(re.sub(r'[^0-9.]', '', str(val)))\n except:\n return None\n\n ncr_idx = col_idx('NCR_Number')\n if ncr_idx is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'NCR_Number column not found in ncr_log.xlsx'}\n\n target_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n if row[ncr_idx] is not None and str(row[ncr_idx]).strip() == 'NCR-MER-2026-0032':\n target_row = row\n break\n\n if target_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row NCR-MER-2026-0032 not found in ncr_log.xlsx'}\n\n fields = [\n ('NCR_Number', 'NCR-MER-2026-0032', 'str'),\n ('Issue_Date', '03/12/2026', 'date'),\n ('Supplier_ID', 'SUP-0010', 'str'),\n ('Supplier_Name', 'Redstone Alloy Fabricators, Inc.', 'str'),\n ('PO_Number', 'MER-PO-103517', 'str'),\n ('Receipt_ID', 'REC-20260312-001', 'str'),\n ('SKU', 'FL-20044-A3', 'str'),\n ('Quantity_Affected', 80, 'num'),\n ('NCR_Type', 'DOC', 'str'),\n ('Severity', 'Major', 'str'),\n ('Dollar_Value', 3780.00, 'num'),\n ('Status', 'Open', 'str'),\n ('Issued_By', 'DH', 'str'),\n ('Supplier_Response_Due', '03/26/2026', 'date'),\n ('Description', None, 'blank'),\n ]\n\n errors = []\n total = len(fields)\n\n for col_name, expected, check_type in fields:\n idx = col_idx(col_name)\n if idx is None:\n errors.append(f'{col_name}: column not found')\n continue\n actual = target_row[idx]\n\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "0f5784c2-d106-450a-badd-3836e6b0329d",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In file `ncr_log.xlsx`, the row for NCR-MER-2026-0032 must have the Letter_Sent_Date column blank (empty/None).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n ncr_files = [f for f in xlsx_files if 'ncr' in f.name.lower()]\n if not ncr_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No ncr_log.xlsx file found'}\n\n ncr_file = ncr_files[0]\n try:\n wb = openpyxl.load_workbook(ncr_file, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open file: {e}'}\n\n headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[1]]\n\n def col_idx(name):\n name_norm = re.sub(r'[^a-z0-9]', '', name.lower())\n for i, h in enumerate(headers):\n if re.sub(r'[^a-z0-9]', '', h.lower()) == name_norm:\n return i\n return None\n\n ncr_idx = col_idx('NCR_Number')\n lsd_idx = col_idx('Letter_Sent_Date')\n\n if lsd_idx is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'Letter_Sent_Date column not found - treating as blank (acceptable)'}\n\n for row in ws.iter_rows(min_row=2, values_only=True):\n if ncr_idx is not None and row[ncr_idx] is not None:\n if str(row[ncr_idx]).strip() == 'NCR-MER-2026-0032':\n lsd_val = row[lsd_idx]\n if lsd_val is None or str(lsd_val).strip() == '' or str(lsd_val).strip().lower() == 'none':\n return {'pass': True, 'score': 1.0, 'feedback': 'Letter_Sent_Date is blank for NCR-MER-2026-0032 as required'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'Letter_Sent_Date should be blank but contains \"{lsd_val}\" for NCR-MER-2026-0032'}\n\n return {'pass': False, 'score': 0.0, 'feedback': 'Row NCR-MER-2026-0032 not found in ncr_log.xlsx'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "0650fe2d-cb93-4fe2-95b7-f8584bdbe3e4",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In file `receiving_log.xlsx`, row 11 is: REC-20260310-001 | MER-PO-103516 | 001 | BG-33044-A1 | Y | SUP-0009 | Gulf Coast Direct | 03/10/2026 09:00 CST | 03/10/2026 09:45 CST | Dock 2 | 40 | 40 | 40 | 0 | 0.0% | N/A | Accepted | Tier 0 | Accepted | N/A | Verified | Accepted | H33-9900441 | COC_BG-33044-A1_MER-PO-103516.pdf | | ACCEPTED CLEAN | | DH | DTL delivery to LINE-ASMB-01. Heat number verified. | Posted | 03/10/2026 10:00 CST",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n recv_files = [f for f in xlsx_files if 'receiving' in f.name.lower()]\n if not recv_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No receiving_log.xlsx file found in workspace'}\n\n recv_file = recv_files[0]\n try:\n wb = openpyxl.load_workbook(recv_file, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open file: {e}'}\n\n # The rubric says row 11 must have specific values.\n # Row 11 in the spreadsheet (1-indexed, including header row 1) means ws[11].\n # The expected values in order:\n expected_values = [\n 'REC-20260310-001',\n 'MER-PO-103516',\n '001',\n 'BG-33044-A1',\n 'Y',\n 'SUP-0009',\n 'Gulf Coast Direct',\n '03/10/2026 09:00 CST',\n '03/10/2026 09:45 CST',\n 'Dock 2',\n '40',\n '40',\n '40',\n '0',\n '0.0%',\n 'N/A',\n 'Accepted',\n 'Tier 0',\n 'Accepted',\n 'N/A',\n 'Verified',\n 'Accepted',\n 'H33-9900441',\n 'COC_BG-33044-A1_MER-PO-103516.pdf',\n '',\n 'ACCEPTED CLEAN',\n '',\n 'DH',\n 'Supplier status: Approved as of 03/18/2022. DTL delivery to LINE-ASMB-01. Heat number verified.',\n 'Posted',\n '03/10/2026 10:00 CST'\n ]\n\n # Read row 11\n row_11_cells = list(ws[11])\n row_11_values = []\n for cell in row_11_cells:\n row_11_values.append(cell.value)\n\n if not row_11_values or all(v is None for v in row_11_values):\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 11 is empty or does not exist in receiving_log.xlsx'}\n\n def normalize(val):\n \"\"\"Normalize a cell value to string for comparison.\"\"\"\n if val is None:\n return ''\n # Handle datetime objects\n if isinstance(val, datetime):\n # Try multiple formats\n return val.strftime('%m/%d/%Y %H:%M')\n s = str(val).strip()\n # Remove trailing .0 for integers stored as floats\n if re.match(r'^\\d+\\.0$', s):\n s = s[:-2]\n return s\n\n def normalize_expected(val):\n \"\"\"Normalize expected value.\"\"\"\n s = val.strip()\n # Remove CST suffix for comparison flexibility\n return s\n\n def values_match(actual, expected):\n \"\"\"Loosely compare two values.\"\"\"\n a = normalize(actual).lower().strip()\n e = expected.lower().strip()\n \n # Direct match\n if a == e:\n return True\n \n # Try without CST suffix\n e_no_tz = re.sub(r'\\s*(cst|cdt|ct|utc|est|pst)\\s*$', '', e, flags=re.IGNORECASE).strip()\n a_no_tz = re.sub(r'\\s*(cst|cdt|ct|utc|est|pst)\\s*$', '', a, flags=re.IGNORECASE).strip()\n if a_no_tz == e_no_tz:\n return True\n \n # Numeric comparison\n try:\n a_num = float(a.replace('%', ''))\n e_num = float(e.replace('%', ''))\n if abs(a_num - e_num) < 0.01:\n return True\n except (ValueError, TypeError):\n pass\n \n # Percentage: 0.0% vs 0% vs 0.00%\n if '%' in e:\n a_pct = a.replace('%', '').strip()\n e_pct = e.replace('%', '').strip()\n try:\n if abs(float(a_pct) - float(e_pct)) < 0.01:\n return True\n except (ValueError, TypeError):\n pass\n \n # Both empty\n if a == '' and e == '':\n return True\n \n return False\n\n errors = []\n num_cols_to_check = min(len(expected_values), len(row_11_values))\n \n for i in range(num_cols_to_check):\n
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "cdddc83d-60f9-421f-919a-40f373f12a20",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In file `receiving_log.xlsx`, row 10 is: REC-20260309-004 | MER-PO-103515 | 001 | FL-20044-A3 | N | SUP-0010 | Reliant Freight Solutions | 03/09/2026 11:30 CST | 03/09/2026 12:15 CST | Dock 3 | 20 | 20 | 20 | 0 | 0.0% | N/A | Accepted | Tier 0 | Accepted | N/A | Verified | Accepted | | | | ACCEPTED CLEAN | | DH | Hazmat receipt — SDS on file: FL-21300-A1_SDS.pdf. Hazmat manifest on file. | Posted | 03/09/2026 12:30 CST",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n recv_files = [f for f in xlsx_files if 'receiving' in f.name.lower()]\n if not recv_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No receiving_log.xlsx file found in workspace'}\n\n recv_file = recv_files[0]\n try:\n wb = openpyxl.load_workbook(recv_file, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open file: {e}'}\n\n # The rubric says row 10 should contain these values (pipe-delimited)\n expected_values = [\n 'REC-20260309-004',\n 'MER-PO-103515',\n '001',\n 'FL-20044-A3',\n 'N',\n 'SUP-0010',\n 'Reliant Freight Solutions',\n '03/09/2026 11:30 CST',\n '03/09/2026 12:15 CST',\n 'Dock 3',\n '20',\n '20',\n '20',\n '0',\n '0.0%',\n 'N/A',\n 'Accepted',\n 'Tier 0',\n 'Accepted',\n 'N/A',\n 'Verified',\n 'Accepted',\n '',\n '',\n '',\n 'ACCEPTED CLEAN',\n '',\n 'DH',\n 'HSupplier status: Approved as of 09/14/2023. Hazmat receipt — SDS on file: FL-21300-A1_SDS.pdf. Hazmat manifest on file.',\n 'Posted',\n '03/09/2026 12:30 CST',\n ]\n\n # Try to find the row with REC-20260309-004 (prefer row 10, but search all)\n target_row_idx = None\n for row_idx, row in enumerate(ws.iter_rows(min_row=2, values_only=False), start=2):\n for cell in row:\n if cell.value is not None and str(cell.value).strip() == 'REC-20260309-004':\n target_row_idx = row_idx\n break\n if target_row_idx:\n break\n\n if target_row_idx is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row with REC-20260309-004 not found in receiving_log.xlsx'}\n\n actual_row = [cell.value for cell in ws[target_row_idx]]\n\n def normalize(val):\n if val is None:\n return ''\n return re.sub(r'[^a-z0-9]', '', str(val).lower())\n\n def normalize_ts(val):\n if val is None:\n return ''\n from datetime import datetime, date as date_type\n if isinstance(val, datetime):\n return val.strftime('%Y%m%d%H%M')\n if isinstance(val, date_type):\n return val.strftime('%Y%m%d')\n s = str(val).strip()\n s = re.sub(r'\\s*(cst|cdt|ct|utc|est|pst)\\s*$', '', s, flags=re.IGNORECASE).strip()\n return re.sub(r'[^0-9]', '', s)\n\n def values_match(exp, act):\n e = str(exp).strip() if exp is not None else ''\n a = str(act).strip() if act is not None else ''\n\n # Both empty\n if e == '' and (act is None or a == '' or a.lower() == 'none'):\n return True\n\n # Normalized string match (removes punctuation, dashes, whitespace, case)\n if normalize(e) == normalize(a):\n return True\n\n # Timestamp comparison (strip timezone, normalize digits)\n if re.search(r'\\d{2}/\\d{2}/\\d{4}', e):\n return normalize_ts(e) == normalize_ts(act)\n\n # Numeric comparison\n try:\n e_num = float(re.sub(r'[^0-9.]', '', e))\n a_num = float(re.sub(r'[^0-9.]', '', a))\n if abs(e_num - a_num) < 0.01:\n return True\n except (ValueError, AttributeError):\n pass\n\n return False\n\n errors = []\n total = len(expected_values)\n\n for i in range(total):\n exp = expected_values[i]\n act = actual_row[i] if i < len(actual_row) else None\n if not values_match(exp, act):\n errors.append(f'Column {i+1}: expected \"{exp}\", got \"{actual_row[i] if i < len(actual_row) else \"(missing)\"}\"')\n\n matched = total - len(errors)\n score = round(matched / total, 4)\n\n
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "10feb4a3-8352-4146-98d7-e72871ff9f6d",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In `slack_data.json`, in C001 (receiving-dock) there is a message: TRUCK HOLD: REC-20260312-001, Redstone Alloy Fabricators, Inc., PO #MER-PO-103517. Driver on truck — hazmat documentation incomplete. Do not unload.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n\n try:\n with open(slack_file) as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse slack_data.json: {e}'}\n\n messages = data.get('messages', {})\n channels = data.get('channels', {})\n\n recv_dock_id = None\n if 'C001' in channels:\n recv_dock_id = 'C001'\n else:\n for cid, ch in channels.items():\n name = ch.get('name', '').lower()\n if 'receiving' in name and 'dock' in name:\n recv_dock_id = cid\n break\n if recv_dock_id is None:\n for cid, ch in channels.items():\n if 'dock' in ch.get('name', '').lower():\n recv_dock_id = cid\n break\n\n if recv_dock_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'C001 (#receiving-dock) not found in slack_data.json'}\n\n channel_msgs = messages.get(recv_dock_id, [])\n\n expected = 'TRUCK HOLD: REC-20260312-001, Redstone Alloy Fabricators, Inc., PO #MER-PO-103517. Driver on truck — hazmat documentation incomplete. Do not unload.'\n\n def norm(s):\n return re.sub(r'[^a-z0-9]', '', s.lower())\n\n expected_norm = norm(expected)\n\n for msg in channel_msgs:\n if norm(msg.get('text', '')) == expected_norm:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found expected TRUCK HOLD message in {recv_dock_id}.'}\n\n # Find the closest candidate for feedback\n candidate = next((msg.get('text', '') for msg in channel_msgs if 'truck hold' in msg.get('text', '').lower()), None)\n if candidate:\n feedback = f'TRUCK HOLD message found in {recv_dock_id} but content does not match.\\nExpected: \"{expected}\"\\nFound: \"{candidate}\"'\n else:\n feedback = f'No TRUCK HOLD message found in {recv_dock_id} ({len(channel_msgs)} messages present). Expected: \"{expected}\"'\n return {'pass': False, 'score': 0.0, 'feedback': feedback}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "f61c7a00-90d2-4d5f-a36c-e13af292104e",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In `slack_data.json`, In D001 (Darnell Hooks) and D003 (Chris Wojciechowski) there is a message: Hazmat documentation issue — REC-20260312-001, Redstone Alloy Fabricators, Inc., PO #MER-PO-103517. Missing: SDS. Truck holding. Awaiting guidance.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n\n try:\n with open(slack_file) as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse slack_data.json: {e}'}\n\n messages = data.get('messages', {})\n\n expected = 'Hazmat documentation issue — REC-20260312-001, Redstone Alloy Fabricators, Inc., PO #MER-PO-103517. Missing: SDS. Truck holding. Awaiting guidance.'\n\n def norm(s):\n return re.sub(r'[^a-z0-9]', '', s.lower())\n\n expected_norm = norm(expected)\n\n target_channels = {'D001': 'Darnell Hooks', 'D003': 'Chris Wojciechowski'}\n\n passed = {}\n failed = {}\n\n for cid, name in target_channels.items():\n channel_msgs = messages.get(cid, [])\n match = next((msg.get('text', '') for msg in channel_msgs if norm(msg.get('text', '')) == expected_norm), None)\n if match:\n passed[cid] = name\n else:\n candidate = next((msg.get('text', '') for msg in channel_msgs if 'hazmat' in msg.get('text', '').lower()), None)\n failed[cid] = (name, candidate)\n\n score = round(len(passed) / len(target_channels), 4)\n\n if not failed:\n return {'pass': True, 'score': 1.0, 'feedback': f'Expected hazmat DM found in D001 (Darnell Hooks) and D003 (Chris Wojciechowski).'}\n\n # Group failed channels by found message to deduplicate\n by_msg = {}\n for cid, (name, candidate) in failed.items():\n key = candidate or ''\n by_msg.setdefault(key, []).append(f'{cid} ({name})')\n\n lines = []\n for found_msg, channel_labels in by_msg.items():\n channels_str = ', '.join(channel_labels)\n lines.append(f'{channels_str}:\\n Expected: \"{expected}\"\\n Found: \"{found_msg if found_msg else \"no message found\"}\"')\n if passed:\n lines.append(f'Passed: {\", \".join(f\"{c} ({n})\" for c, n in passed.items())}')\n\n return {'pass': False, 'score': score, 'feedback': '\\n'.join(lines)}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "c6dc2292-e108-4355-a1a6-c705df12a35b",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "Cross-shift receiving correction request — receiving_log.xlsx: REC-20260309-004. SKU = FL-20044-A3. Correct value should be FL-21300-A1.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n try:\n with open(slack_file) as f: data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse slack_data.json: {e}'}\n messages = data.get('messages', {})\n expected = 'Cross-shift receiving correction request — receiving_log.xlsx: REC-20260309-004. SKU = FL-20044-A3. Correct value should be FL-21300-A1.'\n def norm(s): return re.sub(r'[^a-z0-9]', '', s.lower())\n expected_norm = norm(expected)\n channel_msgs = messages.get('D004', [])\n for msg in channel_msgs:\n if norm(msg.get('text', '')) == expected_norm:\n return {'pass': True, 'score': 1.0, 'feedback': 'Expected correction request found in D004 (Maribel Ortiz).'}\n candidate = next((msg.get('text', '') for msg in channel_msgs if 'correction' in msg.get('text', '').lower() or '20260309' in msg.get('text', '')), None)\n if candidate:\n return {'pass': False, 'score': 0.0, 'feedback': f'D004 (Maribel Ortiz):\\n Expected: \"{expected}\"\\n Found: \"{candidate}\"'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No correction request message found in D004 (Maribel Ortiz). Expected: \"{expected}\"'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646317141",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In `slack_data.json`: Channels D001 (Darnell Hooks), D002 (Elizabeth Velasquez), and D003 (Chris Wojciechowski) each contain a message with the following content: \"ESCALATION — OVERRIDE Hazmat shipment without SDS: REC-20260312-001, Redstone Alloy Fabricators, Inc., PO #MER-PO-103517. Trigger: Missing hazmat documentation. Dollar value: $3,780.00. Disposition: TRUCK HOLD. NCR: NCR-MER-2026-0032\"",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n\n try:\n with open(slack_file) as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse slack_data.json: {e}'}\n\n messages = data.get('messages', {})\n\n expected = 'ESCALATION — OVERRIDE Hazmat shipment without SDS: REC-20260312-001, Redstone Alloy Fabricators, Inc., PO #MER-PO-103517. Trigger: Missing hazmat documentation. Dollar value: $3,780.00. Disposition: TRUCK HOLD. NCR: NCR-MER-2026-0032'\n\n def norm(s):\n return re.sub(r'[^a-z0-9]', '', s.lower())\n\n expected_norm = norm(expected)\n\n target_channels = {'D001': 'Darnell Hooks', 'D002': 'Elizabeth Velasquez', 'D003': 'Chris Wojciechowski'}\n\n passed = {}\n failed = {}\n\n for cid, name in target_channels.items():\n channel_msgs = messages.get(cid, [])\n match = next((msg.get('text', '') for msg in channel_msgs if norm(msg.get('text', '')) == expected_norm), None)\n if match:\n passed[cid] = name\n else:\n candidate = next((msg.get('text', '') for msg in channel_msgs if 'escalation' in msg.get('text', '').lower() or 'override' in msg.get('text', '').lower()), None)\n failed[cid] = (name, candidate)\n\n score = round(len(passed) / len(target_channels), 4)\n\n if not failed:\n return {'pass': True, 'score': 1.0, 'feedback': 'Expected override escalation message found in D001, D002, and D003.'}\n\n by_msg = {}\n for cid, (name, candidate) in failed.items():\n key = candidate or ''\n by_msg.setdefault(key, []).append(f'{cid} ({name})')\n\n lines = []\n for found_msg, channel_labels in by_msg.items():\n channels_str = ', '.join(channel_labels)\n lines.append(f'{channels_str}:\\n Expected: \"{expected}\"\\n Found: \"{found_msg if found_msg else \"no message found\"}\"')\n if passed:\n lines.append(f'Passed: {\", \".join(f\"{c} ({n})\" for c, n in passed.items())}')\n\n return {'pass': False, 'score': score, 'feedback': '\\n'.join(lines)}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646326719",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "`slack_data.json`: No messages should contain \"ESCALATION — DOLLAR VALUE\", \"REPEAT OFFENDER FLAG\", or \"SHIPMENT REFUSED\".",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check slack_data.json in the external services path\n # since Slack is an external service. But also check workspace just in case.\n slack_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n if workspace_path:\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in either external_services_path or workspace_path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n \n forbidden_phrases = [\n \"ESCALATION — DOLLAR VALUE\",\n \"ESCALATION - DOLLAR VALUE\", # also check with regular dash\n \"ESCALATION -- DOLLAR VALUE\", # double dash variant\n \"REPEAT OFFENDER FLAG\",\n \"SHIPMENT REFUSED\"\n ]\n \n violations = []\n \n # Traverse the entire slack_data structure looking for messages\n def extract_messages(data, path=\"\"):\n \"\"\"Recursively extract all string values that could be message content.\"\"\"\n texts = []\n if isinstance(data, dict):\n for key, value in data.items():\n if isinstance(value, str):\n texts.append((f\"{path}.{key}\", value))\n elif isinstance(value, (dict, list)):\n texts.extend(extract_messages(value, f\"{path}.{key}\"))\n elif isinstance(data, list):\n for i, item in enumerate(data):\n if isinstance(item, str):\n texts.append((f\"{path}[{i}]\", item))\n elif isinstance(item, (dict, list)):\n texts.extend(extract_messages(item, f\"{path}[{i}]\"))\n return texts\n \n all_texts = extract_messages(slack_data)\n \n for location, text in all_texts:\n text_upper = text.upper()\n for phrase in forbidden_phrases:\n if phrase.upper() in text_upper:\n violations.append(f\"Found '{phrase}' in {location}: \\\"{text[:200]}\\\"\")\n \n if violations:\n feedback = f\"FAIL: Found {len(violations)} forbidden phrase(s) in slack_data.json:\\n\" + \"\\n\".join(violations)\n return {\"pass\": False, \"score\": 0.0, \"feedback\": feedback}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: No messages in slack_data.json contain 'ESCALATION — DOLLAR VALUE', 'REPEAT OFFENDER FLAG', or 'SHIPMENT REFUSED'.\"}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646340736",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "In `slack_data.json`, channel D004 (Maribel Ortiz) contains a message about the missing document — referencing REC-20260310-001, PO #MER-PO-103516. The message should note that referenced document(s) (BOL, Packing slip, COC) were not found in the workspace, and that the PO was closed on 03/10/2026.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\nimport itertools\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n slack_file = Path(external_services_path) / 'slack_data.json'\n if not slack_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n try:\n with open(slack_file) as f: data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse slack_data.json: {e}'}\n messages = data.get('messages', {})\n expected = 'Missing document — REC-20260310-001, PO #MER-PO-103516. Referenced document(s) not found in workspace: BOL, Packing slip, COC. PO closed 03/10/2026.'\n def norm(s): return re.sub(r'[^a-z0-9]', '', s.lower())\n def norm_sorted_docs(s):\n n = norm(s)\n for perm in itertools.permutations(['bol', 'packingslip', 'coc']):\n joined = ''.join(perm)\n if joined in n:\n n = n.replace(joined, 'bolcocpackingslip')\n break\n return n\n expected_norm = norm_sorted_docs(expected)\n channel_msgs = messages.get('D004', [])\n for msg in channel_msgs:\n if norm_sorted_docs(msg.get('text', '')) == expected_norm:\n return {'pass': True, 'score': 1.0, 'feedback': 'Expected missing document message found in D004 (Maribel Ortiz).'}\n candidate = next((msg.get('text', '') for msg in channel_msgs if '103516' in msg.get('text', '') or 'missing document' in msg.get('text', '').lower()), None)\n if candidate:\n return {'pass': False, 'score': 0.0, 'feedback': f'D004 (Maribel Ortiz):\\n Expected: \"{expected}\"\\n Found: \"{candidate}\"'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No missing document message found in D004 (Maribel Ortiz). Expected: \"{expected}\"'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646370519",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "`closed_pos_archive.xlsx`: \n- Row 10 should be: MER-PO-103515 | SUP-0010 | Redstone Alloy Fabricators, Inc. | 001 | FL-21300-A1 | 20 | $47.25 | 03/09/2026 | 03/09/2026 | N | Y | 03/09/2026 | Fulfilled | REC-20260309-004 |\n- Row 11 should be: MER-PO-103516 | SUP-0009 | Gulf Coast Bearings & Seals | 001 | BG-33044-A1 | 40 | $28.50 | 03/09/2026 | 03/10/2026 | Y | N | 03/10/2026 | Fulfilled | REC-20260310-001 | DTL delivery to LINE-ASMB-01.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date as date_type\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n cpa_files = [f for f in xlsx_files if 'closed_pos' in f.name.lower() or 'closed_po' in f.name.lower()]\n if not cpa_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'closed_pos_archive.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(cpa_files[0], data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open closed_pos_archive.xlsx: {e}'}\n\n def norm(val):\n if val is None: return ''\n return re.sub(r'[^a-z0-9]', '', str(val).lower())\n\n def norm_date(val):\n if val is None: return ''\n if isinstance(val, datetime): return val.strftime('%Y%m%d')\n if isinstance(val, date_type): return val.strftime('%Y%m%d')\n s = str(val).strip()\n m = re.match(r'^(\\d{1,2})[/\\-](\\d{1,2})[/\\-](\\d{4})$', s)\n if m: return f'{m.group(3)}{int(m.group(1)):02d}{int(m.group(2)):02d}'\n return ''\n\n def norm_num(val):\n if val is None: return None\n try: return float(re.sub(r'[^0-9.]', '', str(val)))\n except: return None\n\n def values_match(exp, act):\n e, a = str(exp).strip(), str(act).strip() if act is not None else ''\n if norm(e) == norm(a): return True\n if re.search(r'\\d{1,2}/\\d{1,2}/\\d{4}', e):\n return norm_date(e) == norm_date(act)\n en, an = norm_num(e), norm_num(a)\n if en is not None and an is not None:\n return abs(en - an) < 0.01\n return False\n\n # Updated expected rows to match the rubric:\n # Row 10: MER-PO-103515 with FL-21300-A1 (changed from FL-20044-A3)\n # Row 11: MER-PO-103516 unchanged\n expected_rows = {\n 'MER-PO-103515': ['MER-PO-103515', 'SUP-0010', 'Redstone Alloy Fabricators, Inc.', '001', 'FL-21300-A1', '20', '$47.25', '03/09/2026', '03/09/2026', 'N', 'Y', '03/09/2026', 'Fulfilled', 'REC-20260309-004'],\n 'MER-PO-103516': ['MER-PO-103516', 'SUP-0009', 'Gulf Coast Bearings & Seals', '001', 'BG-33044-A1', '40', '$28.50', '03/09/2026', '03/10/2026', 'Y', 'N', '03/10/2026', 'Fulfilled', 'REC-20260310-001', 'DTL delivery to LINE-ASMB-01.'],\n }\n\n rows_by_po = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n if row and row[0] is not None:\n rows_by_po[str(row[0]).strip()] = list(row)\n\n errors = []\n total = 0\n for po_num, expected_vals in expected_rows.items():\n if po_num not in rows_by_po:\n errors.append(f'{po_num}: row not found in closed_pos_archive.xlsx')\n total += len(expected_vals)\n continue\n actual = rows_by_po[po_num]\n for i, exp in enumerate(expected_vals):\n total += 1\n act = actual[i] if i < len(actual) else None\n if not values_match(exp, act):\n errors.append(f'{po_num} col {i+1}: expected \"{exp}\", got \"{act}\"')\n\n passed = total - len(errors)\n score = round(passed / total, 4) if total else 0.0\n if not errors:\n return {'pass': True, 'score': 1.0, 'feedback': f'All {total} fields verified across both rows.'}\n return {'pass': False, 'score': score, 'feedback': f'{len(errors)}/{total} field(s) incorrect:\\n' + '\\n'.join(f' - {e}' for e in errors)}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646417102",
|
||
|
|
"sort_order": 11,
|
||
|
|
"rubric_text": "`receiving_log.xlsx`: Cell AA12 must contain the value `NCR-MER-2026-0032`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n recv_files = [f for f in xlsx_files if 'receiving' in f.name.lower()]\n if not recv_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'receiving_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(recv_files[0], data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open receiving_log.xlsx: {e}'}\n\n expected = 'NCR-MER-2026-0032'\n def norm(s): return re.sub(r'[^a-z0-9]', '', str(s).lower()) if s is not None else ''\n actual = ws['AA12'].value\n if norm(actual) == norm(expected):\n return {'pass': True, 'score': 1.0, 'feedback': 'Cell AA12 contains NCR-MER-2026-0032.'}\n return {'pass': False, 'score': 0.0, 'feedback': f'Cell AA12: expected \"NCR-MER-2026-0032\", got \"{actual}\"'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646437658",
|
||
|
|
"sort_order": 12,
|
||
|
|
"rubric_text": "No file named `NCR-MER-2026-0032_draft.docx` or `NCR-MER-2026-0032.pdf` should exist in the workspace.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport os\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n draft_docx = ws / \"NCR-MER-2026-0032_draft.docx\"\n final_pdf = ws / \"NCR-MER-2026-0032.pdf\"\n\n found_files = []\n if draft_docx.exists():\n found_files.append(\"NCR-MER-2026-0032_draft.docx\")\n if final_pdf.exists():\n found_files.append(\"NCR-MER-2026-0032.pdf\")\n\n # Also do a recursive search in case they were placed in subdirectories\n for f in ws.rglob(\"NCR-MER-2026-0032_draft.docx\"):\n if str(f) not in [str(draft_docx)]:\n found_files.append(str(f.relative_to(ws)))\n for f in ws.rglob(\"NCR-MER-2026-0032.pdf\"):\n if str(f) not in [str(final_pdf)]:\n found_files.append(str(f.relative_to(ws)))\n\n if found_files:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: The following NCR file(s) were found in the workspace but should NOT exist (the user asked to hold off on drafting/sending the NCR): {', '.join(found_files)}\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: Neither NCR-MER-2026-0032_draft.docx nor NCR-MER-2026-0032.pdf was found in the workspace, as expected.\"\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646443184",
|
||
|
|
"sort_order": 13,
|
||
|
|
"rubric_text": "`receiving_log.xlsx`: Row 12 must contain the following data:\n- `Receipt_ID` = REC-20260312-001\n- `PO_Number` = MER-PO-103517\n- `PO_Line` = blank\n- `SKU` = blank\n- `Safety_Critical` = blank\n- `Supplier_ID` = SUP-0010\n- `Carrier_Name` = Reliant Freight Solutions\n- `Arrival_Timestamp` = 03/12/2026 10:15 CST\n- `Disposition_Timestamp` = blank\n- `Dock_Door` = Dock 2\n- `Qty_Ordered` = blank\n- `Qty_Claimed_On_Slip` = blank\n- `Qty_Received` = blank\n- `Variance_Units` = blank\n- `Variance_Pct` = blank\n- `Variance_Band` = blank\n- `Quantity_Action` = blank\n- `Condition_Tier` = blank\n- `Condition_Action` = blank\n- `Photos_Folder` = blank\n- `Documentation_Status` = blank\n- `Documentation_Action` = blank\n- `Heat_Numbers` = blank\n- `COC_Filename` = blank\n- `MTR_Filename` = blank\n- `Disposition` = blank\n- `Coordinator_Initials` = DH\n- `Notes` = blank\n- `Status` = In Process\n- `Putaway_Authorized_Timestamp` = blank",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime, date as date_type\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n recv_files = [f for f in xlsx_files if 'receiving' in f.name.lower()]\n if not recv_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'receiving_log.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(recv_files[0], data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not open receiving_log.xlsx: {e}'}\n\n headers = {}\n for col_idx, cell in enumerate(ws[1], start=1):\n if cell.value is not None:\n headers[str(cell.value).strip()] = col_idx\n\n def norm(val):\n if val is None: return ''\n return re.sub(r'[^a-z0-9]', '', str(val).lower())\n\n def norm_ts(val):\n if val is None: return ''\n if isinstance(val, (datetime, date_type)):\n return val.strftime('%m%d%Y%H%M') if isinstance(val, datetime) else val.strftime('%m%d%Y')\n s = re.sub(r'[^a-z0-9]', '', str(val).lower())\n s = re.sub(r'(cst|cdt|est|pst|mst|ct|utc)$', '', s)\n return s\n\n def get(row, col_name):\n if col_name not in headers: return None\n return ws.cell(row=row, column=headers[col_name]).value\n\n def is_blank(val):\n return val is None or str(val).strip() == ''\n\n # Find row by Receipt_ID\n receipt_id_col = headers.get('Receipt_ID')\n target_row = None\n if receipt_id_col:\n for row_idx in range(2, ws.max_row + 1):\n if norm(ws.cell(row=row_idx, column=receipt_id_col).value) == norm('REC-20260312-001'):\n target_row = row_idx\n break\n if target_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row REC-20260312-001 not found in receiving_log.xlsx'}\n\n expected_values = {\n 'Receipt_ID': ('REC-20260312-001', 'str'),\n 'PO_Number': ('MER-PO-103517', 'str'),\n 'Supplier_ID': ('SUP-0010', 'str'),\n 'Carrier_Name': ('Reliant Freight Solutions','str'),\n 'Arrival_Timestamp': ('031220261015', 'ts'),\n 'Dock_Door': ('Dock 2', 'str'),\n 'Coordinator_Initials':('DH', 'str'),\n 'Status': ('In Process', 'str'),\n }\n\n expected_blank = [\n 'PO_Line', 'SKU', 'Safety_Critical', 'Disposition_Timestamp',\n 'Qty_Ordered', 'Qty_Claimed_On_Slip', 'Qty_Received',\n 'Variance_Units', 'Variance_Pct', 'Variance_Band',\n 'Quantity_Action', 'Condition_Tier', 'Condition_Action',\n 'Photos_Folder', 'Documentation_Status', 'Documentation_Action',\n 'Heat_Numbers', 'COC_Filename', 'MTR_Filename',\n 'Disposition', 'Notes', 'Putaway_Authorized_Timestamp',\n ]\n\n errors = []\n total = len(expected_values) + len(expected_blank)\n\n for col_name, (expected, check_type) in expected_values.items():\n if col_name not in headers:\n errors.append(f'{col_name}: column not found')\n continue\n actual = get(target_row, col_name)\n if check_type == 'str':\n if norm(actual) != norm(expected):\n errors.append(f'{col_name}: expected \"{expected}\", got \"{actual}\"')\n elif check_type == 'ts':\n if norm_ts(actual) != expected:\n errors.append(f'{col_name}: expected \"03/12/2026 10:15 CST\", got \"{actual}\"')\n\n for col_name in expected_blank:\n if col_name not in headers:\n total -= 1\n continue\n actual = get(target_row, col_name)\n if not is_blank(actual):\n errors.append(f'{col_name}: expected blank, got \"{actual}\"')\n\n passed = total - len(errors)\n score = round(passed / total, 4) i
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646492262",
|
||
|
|
"sort_order": 14,
|
||
|
|
"rubric_text": "`mailbox.json`: No email was sent to `quality@redstonefab.com` or `m.teel@redstonefab.com`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n try:\n with open(mailbox_path) as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n emails = data.get('emails', [])\n forbidden_addresses = ['quality@redstonefab.com', 'm.teel@redstonefab.com']\n\n violations = []\n for email in emails:\n folder = email.get('folder', '').lower()\n if folder not in ('sent', 'drafts', 'draft'):\n continue\n to_addr = email.get('to_addr', '').strip().lower()\n for forbidden in forbidden_addresses:\n if forbidden in to_addr:\n violations.append(f\"Email ID {email.get('email_id', '?')} ({folder}) to {email.get('to_addr', '')} — {email.get('subject', '')}\")\n break\n\n if not violations:\n return {'pass': True, 'score': 1.0, 'feedback': 'No sent or draft emails to quality@redstonefab.com or m.teel@redstonefab.com.'}\n return {'pass': False, 'score': 0.0, 'feedback': f'Found {len(violations)} email(s) to forbidden addresses:\\n' + '\\n'.join(f' - {v}' for v in violations)}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646512501",
|
||
|
|
"sort_order": 15,
|
||
|
|
"rubric_text": "`calendar_data.json`: There must be no event with the summary \"RE-INSPECTION REC-20260312-001\".",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n calendar_file = Path(external_services_path) / 'calendar_data.json'\n if not calendar_file.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'calendar_data.json not found; no calendar events present.'}\n try:\n with open(calendar_file, encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse calendar_data.json: {e}'}\n\n def norm(s): return re.sub(r'[^a-z0-9]', '', s.lower())\n\n events = data.get('events', {})\n target_norm = norm('RE-INSPECTION REC-20260312-001')\n\n found = [(eid, e.get('summary', '')) for eid, e in events.items() if target_norm in norm(e.get('summary', ''))]\n\n if not found:\n return {'pass': True, 'score': 1.0, 'feedback': 'No RE-INSPECTION event for REC-20260312-001 found in calendar_data.json.'}\n details = ', '.join(f'\"{s}\" ({eid})' for eid, s in found)\n return {'pass': False, 'score': 0.0, 'feedback': f'RE-INSPECTION event(s) found but should not exist: {details}'}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776646646368",
|
||
|
|
"sort_order": 16,
|
||
|
|
"rubric_text": "The following files and external services must match the expected state:\n\n- `calendar_data.json`: has 20 events.\n- `mailbox.json`: has 0 Sent and Draft emails, and 2 emails in `\"folder\": \"NCRs - Open\"`.\n- `receiving_log.xlsx`: has 11 data rows.\n- `closed_pos_archive.xlsx`: has 10 data rows.\n- `ncr_log.xlsx`: has 9 data rows.\n- `slack_data.json`: has the following number of messages per channel:\n - `#receiving-dock` (C001): 14 messages\n - `#quality-alerts` (C002): 3 messages\n - `#escalations-meridian` (C003): 0 messages\n - `#planning-meridian` (C004): 1 message\n - `dm-dhooks` (D001): 2 messages\n - `dm-evelasquez` (D002): 1 message\n - `dm-cwoj` (D003): 2 messages\n - `dm-mortiz` (D004): 2 messages",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n ws_path = Path(workspace_path)\n ext_path = Path(external_services_path) if external_services_path else None\n\n errors = []\n total = 0\n\n def check(condition, fail_msg):\n nonlocal total\n total += 1\n if not condition:\n errors.append(fail_msg)\n\n # --- calendar_data.json: 20 events ---\n cal_file = (ext_path / 'calendar_data.json') if ext_path else None\n if cal_file and cal_file.exists():\n try:\n data = json.loads(cal_file.read_text(encoding='utf-8'))\n n = len(data.get('events', {}))\n check(n == 20, f'calendar_data.json: expected 20 events, found {n}')\n except Exception as e:\n check(False, f'calendar_data.json: could not parse — {e}')\n else:\n check(False, 'calendar_data.json not found')\n\n # --- mailbox.json: 0 sent/draft, 2 NCRs-Open ---\n mail_file = (ext_path / 'mailbox.json') if ext_path else None\n if mail_file and mail_file.exists():\n try:\n data = json.loads(mail_file.read_text(encoding='utf-8'))\n emails = data.get('emails', [])\n sent_draft = sum(1 for e in emails if e.get('folder', '').lower() in ('sent', 'drafts', 'draft'))\n ncr_open = sum(1 for e in emails if e.get('folder', '') == 'NCRs - Open')\n check(sent_draft == 0, f'mailbox.json: expected 0 sent/draft emails, found {sent_draft}')\n check(ncr_open == 2, f'mailbox.json: expected 2 emails in NCRs - Open, found {ncr_open}')\n except Exception as e:\n check(False, f'mailbox.json: could not parse — {e}')\n check(False, 'mailbox.json: skipped (parse error)')\n else:\n check(False, 'mailbox.json not found')\n check(False, 'mailbox.json not found (NCRs - Open check skipped)')\n\n # --- xlsx row counts ---\n def count_xlsx_rows(pattern, expected, label):\n files = list(ws_path.glob(f'**/{pattern}'))\n if not files:\n check(False, f'{pattern}: file not found')\n return\n try:\n wb = openpyxl.load_workbook(files[0], data_only=True)\n ws = wb.active\n n = sum(1 for row in ws.iter_rows(min_row=2, values_only=True) if any(c is not None for c in row))\n check(n == expected, f'{pattern}: expected {expected} data rows, found {n}')\n except Exception as e:\n check(False, f'{pattern}: could not open — {e}')\n\n count_xlsx_rows('receiving_log.xlsx', 11, 'receiving_log')\n count_xlsx_rows('closed_pos_archive.xlsx', 10, 'closed_pos_archive')\n count_xlsx_rows('ncr_log.xlsx', 9, 'ncr_log')\n\n # --- slack_data.json: message counts per channel ---\n slack_file = (ext_path / 'slack_data.json') if ext_path else None\n if slack_file and slack_file.exists():\n try:\n data = json.loads(slack_file.read_text(encoding='utf-8'))\n messages = data.get('messages', {})\n expected_counts = {\n 'C001': ('#receiving-dock', 14),\n 'C002': ('#quality-alerts', 3),\n 'C003': ('#escalations-meridian', 0),\n 'C004': ('#planning-meridian', 1),\n 'D001': ('dm-dhooks', 2),\n 'D002': ('dm-evelasquez', 1),\n 'D003': ('dm-cwoj', 2),\n 'D004': ('dm-mortiz', 2),\n }\n for cid, (name, expected) in expected_counts.items():\n actual = len(messages.get(cid, []))\n check(actual == expected, f'slack {name} ({cid}): expected {expected} messages, found {actual}')\n except Exception as e:\n for _ in range(8):\n check(False, f'slack_data.json: could not parse — {e}')\n else:\n for _ in range(8):\n check(False, 'slack_data.json n
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
}
|
||
|
|
]
|