143 lines
48 KiB
JSON
143 lines
48 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "e9e30a9b-fdae-4d1a-b35a-bce846762f57",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In `mailbox.json`, no sent emails must exist with `to_addr` set to `facilities@crestwood.edu`",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n violations = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\" and \"facilities@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} sent email(s) to facilities@crestwood.edu, expected none.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails to facilities@crestwood.edu found.\"}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "e7bfa402-fa2a-4fc5-bdf6-b5099fa59d0a",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In `mailbox.json`, the email from `facilities@crestwood.edu` must be in the `general` folder",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n from_facilities = [e for e in emails if \"facilities@crestwood.edu\" in str(e.get(\"from_addr\", \"\")).lower()]\n if not from_facilities:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email from facilities@crestwood.edu found.\"}\n for e in from_facilities:\n folder = e.get(\"folder\", \"\").lower()\n if \"general\" not in folder:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email from facilities@crestwood.edu is in folder '{e.get('folder')}', expected 'general'.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email from facilities@crestwood.edu is in the general folder.\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "e7f91235-98e4-4389-95cf-2b7767dea2f1",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In `slack_data.json`, channel `#hr-general` must contain a message with text `[OUT OF SCOPE] - HR office light bulb out – Main Campus rm 214`",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n hr_general_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-general\":\n hr_general_id = cid\n break\n if hr_general_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-general not found.\"}\n messages = data.get(\"messages\", {}).get(hr_general_id, [])\n\n def normalize_dashes(s):\n # Collapse any hyphen/en-dash/em-dash/minus to a single canonical '-'\n return (s.replace(\"\\u2014\", \"-\")\n .replace(\"\\u2013\", \"-\")\n .replace(\"\\u2212\", \"-\")\n .replace(\"-\", \"-\"))\n\n target = normalize_dashes(\"[OUT OF SCOPE] - HR office light bulb out - Main Campus rm 214\").lower()\n for msg in messages:\n text = normalize_dashes(msg.get(\"text\", \"\")).lower()\n if target in text:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required message in #hr-general: '{msg.get('text')}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Message '[OUT OF SCOPE] - HR office light bulb out - Main Campus rm 214' (any dash variant) not found in #hr-general. Messages found: {[m.get('text') for m in messages]}\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "6059ef2b-9c84-4307-8785-3c9e1948af51",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "In `mailbox.json`, the email from `l.fontaine@crestwood.edu` must be in the `leave` folder",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n from_fontaine = [e for e in emails if \"l.fontaine@crestwood.edu\" in str(e.get(\"from_addr\", \"\")).lower()]\n if not from_fontaine:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email from l.fontaine@crestwood.edu found.\"}\n for e in from_fontaine:\n folder = e.get(\"folder\", \"\").lower()\n if \"leave\" not in folder:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email from l.fontaine@crestwood.edu is in folder '{e.get('folder')}', expected 'leave'.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email from l.fontaine@crestwood.edu is in the leave folder.\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "fa6332bb-e59b-433e-b135-335bcb794eef",
|
|||
|
|
"sort_order": 4,
|
|||
|
|
"rubric_text": "In `mailbox.json`, a sent email to `l.fontaine@crestwood.edu` must exist with body containing PTO balance of 68 hours, accrual rate of 4.62 hours per pay period, and contact info for Patricia Huang at p.huang@crestwood.edu",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n candidates = [e for e in emails if \"l.fontaine@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to l.fontaine@crestwood.edu found.\"}\n required_phrases = [\"68 hours\", \"4.62\", \"p.huang@crestwood.edu\", \"04/07/2026\"]\n for e in candidates:\n body = e.get(\"body_text\", \"\").lower()\n if all(phrase.lower() in body for phrase in required_phrases):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email to l.fontaine@crestwood.edu found with required PTO balance content.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email to l.fontaine@crestwood.edu found but missing required content. Required: {required_phrases}. Bodies found: {[e.get('body_text','')[:200] for e in candidates]}\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "4ed6f0e7-1d75-45cc-a70d-a53741123f63",
|
|||
|
|
"sort_order": 5,
|
|||
|
|
"rubric_text": "In `mailbox.json`, a sent email to `p.huang@crestwood.edu` with CC `t.salazar@crestwood.edu` with subject `I-9 Urgent Reminder — Salazar, CU-1026` must exist with body mentioning Tomás Salazar, CU-1026, 04/08/2026 federal deadline, and Section 2 of I-9",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n\n # Find emails sent to p.huang@crestwood.edu\n candidates = []\n for e in emails:\n to_addr = e.get(\"to_addr\", \"\") or e.get(\"to\", \"\")\n if isinstance(to_addr, list):\n to_lower = [a.lower() for a in to_addr]\n else:\n to_lower = [str(to_addr).lower()]\n if any(\"p.huang@crestwood.edu\" in a for a in to_lower):\n candidates.append(e)\n\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to p.huang@crestwood.edu found.\"}\n\n # Filter for I-9 / Salazar / CU-1026 related subject\n salazar_emails = []\n for e in candidates:\n subj = e.get(\"subject\", \"\").lower()\n if \"salazar\" in subj or \"cu-1026\" in subj or \"i-9\" in subj or \"i9\" in subj:\n salazar_emails.append(e)\n\n if not salazar_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No I-9/Salazar-related email found among emails to p.huang@crestwood.edu. Subjects found: {[e.get('subject') for e in candidates]}\"}\n\n for e in salazar_emails:\n body = e.get(\"body_text\", \"\") or e.get(\"body\", \"\") or \"\"\n subject = e.get(\"subject\", \"\")\n full_text = (subject + \" \" + body).lower()\n\n # Check CC includes t.salazar@crestwood.edu\n cc = e.get(\"cc_addr\", \"\") or e.get(\"cc\", \"\") or \"\"\n if isinstance(cc, list):\n cc_lower = [a.lower() for a in cc]\n else:\n cc_lower = [str(cc).lower()]\n has_cc = any(\"t.salazar@crestwood.edu\" in c for c in cc_lower)\n\n # Check body mentions: Salazar (or Tomás Salazar), CU-1026, April 8 2026 deadline, Section 2\n has_salazar = \"salazar\" in full_text\n has_cu1026 = \"cu-1026\" in full_text or \"cu1026\" in full_text\n\n # Accept various date formats for April 8, 2026\n date_patterns = [\n \"04/08/2026\", \"4/8/2026\",\n ]\n has_date = any(p in full_text for p in date_patterns)\n\n # Section 2 check - be lenient\n has_section2 = \"section 2\" in full_text or \"section ii\" in full_text or \"section two\" in full_text or \"section-2\" in full_text\n\n if has_salazar and has_cu1026 and has_date and has_section2:\n if not has_cc:\n return {\"pass\": True, \"score\": 0.8, \"feedback\": f\"Found I-9 urgent reminder email to p.huang@crestwood.edu for Salazar CU-1026 with required content, but CC to t.salazar@crestwood.edu not confirmed. CC field: {cc}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found I-9 urgent reminder email to p.huang@crestwood.edu with CC t.salazar@crestwood.edu for Salazar CU-1026 with all required content (Tomás Salazar, CU-1026, April 8 2026 deadline, Section 2).\"}\n\n # Provide detailed failure info\n details = []\n for e in salazar_emails:\n body = e.get(\"body_text\", \"\") or e.get(\"body\", \"\") or \"\"\n full_text = (e.get(\"subject\", \"\") + \" \" + body).lower()\n cc = e.get(\"cc_addr\", \"\") or e.get(\"cc\", \"\") or \"\"\n missing = []\n if \"salazar\" not in full_text:\n missing.append(\"Salazar\")\n if \"cu-1026\" not in full_text and \"cu1026\" not in full_text:\n missing.append(\"CU-1026\")\n date_patterns = [\n \"april 8\", \"april 08\", \"04/08/2026\", \"4/8/2026\",\n \"2026-04-08\", \"apr
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "559efc43-ae89-4956-8045-90ca9cb46fbb",
|
|||
|
|
"sort_order": 6,
|
|||
|
|
"rubric_text": "In `benefits_enrollment.xlsx`, cell C26 must be set to `Default Enrollment` and cell D26 must be set to `05/01/2026`",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / \"benefits_enrollment.xlsx\"\n if not wb_path.exists():\n # Try searching\n matches = list(Path(workspace_path).glob(\"**/benefits_enrollment.xlsx\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"benefits_enrollment.xlsx not found in workspace.\"}\n wb_path = matches[0]\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n c26 = ws[\"C26\"].value\n d26 = ws[\"D26\"].value\n c26_str = str(c26).strip() if c26 is not None else \"\"\n d26_str = str(d26).strip() if d26 is not None else \"\"\n c26_ok = \"default enrollment\" in c26_str.lower()\n d26_ok = \"05/01/2026\" in d26_str or d26_str == \"05/01/2026\"\n # Also check if d26 is a date object\n import datetime\n if hasattr(d26, 'month') and hasattr(d26, 'year'):\n d26_ok = d26.month == 5 and d26.year == 2026 and d26.day == 1\n if c26_ok and d26_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"C26='{c26}' (Default Enrollment) and D26='{d26}' (05/01/2026) correctly set.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"C26='{c26}' (expected 'Default Enrollment', ok={c26_ok}), D26='{d26}' (expected 05/01/2026, ok={d26_ok}).\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "4323ac49-9f39-4121-9b6d-13ab81d30ed0",
|
|||
|
|
"sort_order": 7,
|
|||
|
|
"rubric_text": "In `onboarding_checklist.xlsx`, cell E15 must be set to `05/01/2026` (Benefits Enrolled date)",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / \"onboarding_checklist.xlsx\"\n if not wb_path.exists():\n matches = list(Path(workspace_path).glob(\"**/onboarding_checklist.xlsx\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"onboarding_checklist.xlsx not found in workspace.\"}\n wb_path = matches[0]\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n e15 = ws[\"E15\"].value\n e15_str = str(e15).strip() if e15 is not None else \"\"\n ok = \"05/01/2026\" in e15_str\n if hasattr(e15, 'month') and hasattr(e15, 'year'):\n ok = e15.month == 5 and e15.year == 2026 and e15.day == 1\n if ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"E15='{e15}' correctly set to 05/01/2026 for Benefits Enrolled.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"E15='{e15}' does not match expected 05/01/2026.\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "f2b2962e-5acc-4346-a75d-c52075fa9e35",
|
|||
|
|
"sort_order": 8,
|
|||
|
|
"rubric_text": "In `employee_roster.xlsx`, cell L23 must be set to `Active`",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / \"employee_roster.xlsx\"\n if not wb_path.exists():\n matches = list(Path(workspace_path).glob(\"**/employee_roster.xlsx\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"employee_roster.xlsx not found in workspace.\"}\n wb_path = matches[0]\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n l23 = ws[\"L23\"].value\n l23_str = str(l23).strip() if l23 is not None else \"\"\n if l23_str.lower() == \"active\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"L23='{l23}' correctly set to Active.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"L23='{l23}' does not match expected 'Active'.\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "efd47f2e-f94d-4e44-b54b-7b67ad1ba7fe",
|
|||
|
|
"sort_order": 9,
|
|||
|
|
"rubric_text": "In `slack_data.json`, channel `#hr-onboarding` must contain a message with exact text `ONBOARDING COMPLETE: William Trent, CU-1022, 04/07/2026`",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n hr_onboarding_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-onboarding\":\n hr_onboarding_id = cid\n break\n if hr_onboarding_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-onboarding not found.\"}\n messages = data.get(\"messages\", {}).get(hr_onboarding_id, [])\n target = \"ONBOARDING COMPLETE: William Trent, CU-1022, 04/07/2026\"\n for msg in messages:\n if target.lower() in msg.get(\"text\", \"\").lower():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required message in #hr-onboarding: '{msg.get('text')}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Message '{target}' not found in #hr-onboarding. Messages: {[m.get('text') for m in messages]}\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "62bb6384-ef1c-4030-b4d3-3dd3ff76d725",
|
|||
|
|
"sort_order": 10,
|
|||
|
|
"rubric_text": "In `mailbox.json`, a sent email to `w.trent@crestwood.edu` with subject `Default Benefits Enrollment Confirmation — Trent, CU-1022` must exist with body mentioning default enrollment, 04/02/2026 deadline, and 05/01/2026 effective date",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n candidates = [e for e in emails if \"w.trent@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to w.trent@crestwood.edu found.\"}\n subject_match = [e for e in candidates if \"cu-1022\" in e.get(\"subject\", \"\").lower() or \"trent\" in e.get(\"subject\", \"\").lower()]\n required_phrases = [\"04/02/2026\", \"05/01/2026\", \"default\"]\n for e in subject_match:\n body = e.get(\"body_text\", \"\")\n if all(phrase.lower() in body.lower() for phrase in required_phrases):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found default benefits enrollment confirmation email to w.trent@crestwood.edu with required content.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Default benefits enrollment confirmation email to w.trent@crestwood.edu not found or missing required content. Required: {required_phrases}. Subjects: {[e.get('subject') for e in candidates]}\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "97c5f3d5-b336-4a10-b782-d15d710b62d8",
|
|||
|
|
"sort_order": 11,
|
|||
|
|
"rubric_text": "In `mailbox.json`, a sent email to `p.huang@crestwood.edu` with subject `FMLA Deadline Passed — CU-1023` must exist with body mentioning Meghan Cross FMLA Certification overdue",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n candidates = [e for e in emails if \"p.huang@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to p.huang@crestwood.edu found.\"}\n fmla_emails = [e for e in candidates if \"fmla\" in e.get(\"subject\", \"\").lower() and \"cu-1023\" in e.get(\"subject\", \"\").lower()]\n if not fmla_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No FMLA CU-1023 email to p.huang@crestwood.edu found. Subjects: {[e.get('subject') for e in candidates]}\"}\n for e in fmla_emails:\n body = e.get(\"body_text\", \"\")\n if \"cross\" in body.lower() or \"meghan\" in body.lower():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found FMLA Deadline Passed email to p.huang@crestwood.edu for CU-1023 (Meghan Cross).\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FMLA CU-1023 email to p.huang@crestwood.edu found but body missing 'Cross'/'Meghan'. Bodies: {[e.get('body_text','')[:200] for e in fmla_emails]}\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "342775db-78be-4cd8-a801-e523a0286686",
|
|||
|
|
"sort_order": 12,
|
|||
|
|
"rubric_text": "In `mailbox.json`, a sent email to `m.cross@crestwood.edu` with subject `FMLA Certification Overdue — Cross, CU-1023` OR `Template 18 — Cross, CU-1023` OR `Template 18 – FMLA Certification Overdue — Cross, CU-1023` must exist with body mentioning due date 04/07/2026, FMLA Medical Certification Form, and contact info for Patricia Huang",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n # Find emails sent to m.cross@crestwood.edu\n candidates = [e for e in emails if \"m.cross@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to m.cross@crestwood.edu found.\"}\n\n # Check subject matches one of the allowed patterns (loose matching)\n subject_keywords = [\"fmla\", \"template 18\", \"cross\", \"cu-1023\"]\n\n # Check body for required content\n # Due date: 04/07/2026 in various formats\n date_patterns = [\"04/07/2026\", \"4/7/2026\"]\n # FMLA Medical Certification Form\n fmla_form_patterns = [\"fmla medical certification\", \"fmla certification\", \"medical certification form\"]\n # Contact info for Patricia Huang - could be email, phone, or just name\n patricia_patterns = [\"patricia huang\", \"p.huang@crestwood.edu\", \"p. huang\"]\n\n for e in candidates:\n body = (e.get(\"body_text\", \"\") or \"\").lower()\n subject = (e.get(\"subject\", \"\") or \"\").lower()\n combined = subject + \" \" + body\n\n # Subject check: must relate to FMLA or Template 18 and Cross/CU-1023\n subject_ok = False\n if (\"fmla\" in subject or \"template 18\" in subject) and (\"cross\" in subject or \"cu-1023\" in subject):\n subject_ok = True\n\n if not subject_ok:\n continue\n\n # Date check\n date_ok = any(dp in combined for dp in date_patterns)\n\n # FMLA form check\n fmla_form_ok = any(fp in combined for fp in fmla_form_patterns)\n\n # Patricia Huang contact info check\n patricia_ok = any(pp in combined for pp in patricia_patterns)\n\n if date_ok and fmla_form_ok and patricia_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found FMLA Certification Overdue email to m.cross@crestwood.edu with required content. Subject: {e.get('subject')}\"}\n\n # Collect what's missing for feedback\n missing = []\n if not date_ok:\n missing.append(\"due date (04/07/2026)\")\n if not fmla_form_ok:\n missing.append(\"FMLA Medical Certification Form reference\")\n if not patricia_ok:\n missing.append(\"Patricia Huang contact info\")\n\n # If we get here, no email matched fully\n all_subjects = [e.get('subject', '') for e in candidates]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FMLA Certification Overdue email to m.cross@crestwood.edu not found or missing required content. Need: due date 04/07/2026, FMLA Medical Certification Form mention, and Patricia Huang contact info. Candidate subjects: {all_subjects}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "543a04a0-b8fa-4f83-96f1-70823791a25d",
|
|||
|
|
"sort_order": 13,
|
|||
|
|
"rubric_text": "In `slack_data.json`, channel `#hr-leave-requests` must contain a message with text `FMLA CERTIFICATION OVERDUE: Meghan Cross, CU-1023, Due: 04/07/2026` and mentioning overdue",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n hr_leave_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-leave-requests\":\n hr_leave_id = cid\n break\n if hr_leave_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-leave-requests not found.\"}\n messages = data.get(\"messages\", {}).get(hr_leave_id, [])\n required_phrases = [\"FMLA CERTIFICATION OVERDUE\", \"Meghan Cross\", \"CU-1023\", \"04/07/2026\"]\n for msg in messages:\n text = msg.get(\"text\", \"\")\n if all(phrase.lower() in text.lower() for phrase in required_phrases):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required FMLA overdue message in #hr-leave-requests: '{text}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Required FMLA overdue message not found in #hr-leave-requests. Required phrases: {required_phrases}. Messages: {[m.get('text') for m in messages]}\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "6b40093b-ab7a-46b8-9129-bcd2c58d223a",
|
|||
|
|
"sort_order": 14,
|
|||
|
|
"rubric_text": "In `jira_state.json`, issue `HR-7` must have comments containing all required entries for steps 11.3 and 11.4, and description appended with `Status — CLOSED`",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n def adf_to_text(node):\n \"\"\"Flatten an Atlassian Document Format node (dict/list) to plain text. Passthrough for strings.\"\"\"\n if node is None:\n return \"\"\n if isinstance(node, str):\n return node\n out = []\n def walk(n):\n if isinstance(n, dict):\n if n.get(\"type\") == \"text\":\n out.append(n.get(\"text\", \"\"))\n for c in n.get(\"content\", []) or []:\n walk(c)\n if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n out.append(\"\\n\")\n elif isinstance(n, list):\n for x in n:\n walk(x)\n walk(node)\n return \"\".join(out).strip()\n\n # Navigate to issues\n issues = data.get(\"issues\", data) if isinstance(data, dict) else data\n\n trent_issue = None\n matched_key = None\n if isinstance(issues, dict):\n for key, val in issues.items():\n if \"trent\" in key.lower() and \"onboard\" in key.lower():\n trent_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n if trent_issue is None:\n for key, val in issues.items():\n combined = (key + \" \" + json.dumps(val, default=str)).lower()\n if \"trent\" in combined and \"onboard\" in combined:\n trent_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict):\n item_str = json.dumps(item, default=str).lower()\n if \"trent\" in item_str and \"onboard\" in item_str:\n trent_issue = item\n matched_key = item.get(\"key\", \"list_item\")\n break\n\n if trent_issue is None:\n all_keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A (list)\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-7 not found in jira_state.json. Available keys: {all_keys}\"}\n\n # Gather comments. Handle both schemas:\n # (a) comments nested inside the issue: issue[\"comments\"] = [...]\n # (b) comments stored in a parallel top-level dict: data[\"comments\"][issue_key] = [...]\n # Each comment's body may be a plain string OR an ADF dict.\n comments_raw = []\n nested = trent_issue.get(\"comments\")\n if isinstance(nested, list):\n comments_raw.extend(nested)\n if isinstance(data, dict):\n parallel = data.get(\"comments\", {})\n if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n comments_raw.extend(parallel[matched_key])\n\n comments_text_parts = []\n for c in comments_raw:\n if isinstance(c, dict):\n for field in (\"body\", \"text\", \"content\"):\n if field in c:\n comments_text_parts.append(adf_to_text(c[field]))\n else:\n comments_text_parts.append(str(c))\n comments_text = \" \".join(comments_text_parts)\n comments_text_lower = comments_text.lower()\n\n # Description. Handle both schemas:\n # (a) issue[\"description\"] directly (string or ADF)\n # (b) issue[\"fields\"][\"description\"] (string or ADF)\n description = trent_issue.ge
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "fd6fcf96-a025-43ef-b1ec-d6bd8023ec7d",
|
|||
|
|
"sort_order": 15,
|
|||
|
|
"rubric_text": "In `jira_state.json`, issue `HR-1` must have comments for Template 18 sent to m.cross@crestwood.edu, slack posted to #hr-leave-requests, email forwarded to p.huang@crestwood.edu, and waiting on Patricia Huang",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n def adf_to_text(node):\n if node is None:\n return \"\"\n if isinstance(node, str):\n return node\n out = []\n def walk(n):\n if isinstance(n, dict):\n if n.get(\"type\") == \"text\":\n out.append(n.get(\"text\", \"\"))\n for c in n.get(\"content\", []) or []:\n walk(c)\n if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n out.append(\"\\n\")\n elif isinstance(n, list):\n for x in n:\n walk(x)\n walk(node)\n return \"\".join(out).strip()\n\n issues = data.get(\"issues\", data) if isinstance(data, dict) and \"issues\" in data else data\n\n cross_issue = None\n matched_key = None\n if isinstance(issues, dict):\n for key, val in issues.items():\n if \"cross\" in key.lower() and \"leave\" in key.lower():\n cross_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict):\n s = json.dumps(item, default=str).lower()\n if \"cross\" in s and \"leave\" in s:\n cross_issue = item\n matched_key = item.get(\"key\", \"list_item\")\n break\n\n if cross_issue is None:\n keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-1 not found. Keys: {keys}\"}\n\n comments_raw = []\n nested = cross_issue.get(\"comments\")\n if isinstance(nested, list):\n comments_raw.extend(nested)\n if isinstance(data, dict):\n parallel = data.get(\"comments\", {})\n if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n comments_raw.extend(parallel[matched_key])\n\n parts = []\n for c in comments_raw:\n if isinstance(c, dict):\n for field in (\"body\", \"text\", \"content\"):\n if field in c:\n parts.append(adf_to_text(c[field]))\n else:\n parts.append(str(c))\n comments_text = \" \".join(parts)\n ct = comments_text.lower()\n\n required_fragments = [\"template 18\", \"m.cross@crestwood.edu\", \"hr-leave-requests\", \"p.huang@crestwood.edu\"]\n missing = [f for f in required_fragments if f.lower() not in ct]\n if missing:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"HR-1 (key: {matched_key}) missing fragments: {missing}. Found {len(comments_raw)} comments. Text: '{comments_text[:400]}'\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"HR-1 (key: {matched_key}) has required fragments across {len(comments_raw)} comments.\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "0b596b96-db7e-445f-a83b-05ef87e66771",
|
|||
|
|
"sort_order": 16,
|
|||
|
|
"rubric_text": "In `jira_state.json`, issue `HR-6` must have a comment mentioning Template 6 sent to p.huang@crestwood.edu and step 11.2",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n def adf_to_text(node):\n \"\"\"Flatten Atlassian Document Format (dict/list) to plain text; passthrough for strings.\"\"\"\n if node is None:\n return \"\"\n if isinstance(node, str):\n return node\n out = []\n def walk(n):\n if isinstance(n, dict):\n if n.get(\"type\") == \"text\":\n out.append(n.get(\"text\", \"\"))\n for c in n.get(\"content\", []) or []:\n walk(c)\n if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n out.append(\"\\n\")\n elif isinstance(n, list):\n for x in n:\n walk(x)\n walk(node)\n return \"\".join(out).strip()\n\n issues = data.get(\"issues\", data) if isinstance(data, dict) and \"issues\" in data else data\n\n salazar_issue = None\n matched_key = None\n if isinstance(issues, dict):\n for key, val in issues.items():\n if \"salazar\" in key.lower() and \"onboard\" in key.lower():\n salazar_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n if salazar_issue is None:\n for key, val in issues.items():\n combined = (key + \" \" + json.dumps(val, default=str)).lower()\n if \"salazar\" in combined and \"onboard\" in combined:\n salazar_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict):\n s = json.dumps(item, default=str).lower()\n if \"salazar\" in s and \"onboard\" in s:\n salazar_issue = item\n matched_key = item.get(\"key\", \"list_item\")\n break\n\n if salazar_issue is None:\n keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-6 not found. Keys: {keys}\"}\n\n # Collect comments from both possible schemas.\n comments_raw = []\n nested = salazar_issue.get(\"comments\")\n if isinstance(nested, list):\n comments_raw.extend(nested)\n if isinstance(data, dict):\n parallel = data.get(\"comments\", {})\n if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n comments_raw.extend(parallel[matched_key])\n\n parts = []\n for c in comments_raw:\n if isinstance(c, dict):\n for field in (\"body\", \"text\", \"content\"):\n if field in c:\n parts.append(adf_to_text(c[field]))\n else:\n parts.append(str(c))\n comments_text = \" \".join(parts)\n ct = comments_text.lower()\n\n required_fragments = [\"11.2\", \"Template 6\", \"p.huang@crestwood.edu\"]\n missing = [f for f in required_fragments if f.lower() not in ct]\n if missing:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"HR-6 (key: {matched_key}) missing comment fragments: {missing}. Found {len(comments_raw)} comments. Text: '{comments_text[:400]}'\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"HR-6 (key: {matched_key}) has required comment for Template 6 to p.huang@crestwood.edu across {len(
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "0968f22e-9afc-4fa9-8baf-f8ed52566e11",
|
|||
|
|
"sort_order": 17,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be exactly 11 sent emails (emails with folder='Sent')",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n if len(sent) == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 11 sent emails. Subjects: {[e.get('subject') for e in sent]}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 11 sent emails, found {len(sent)}. Subjects: {[e.get('subject') for e in sent]}\"}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "9fa6493a-d029-4e8f-a6ab-fdade61b8c30",
|
|||
|
|
"sort_order": 18,
|
|||
|
|
"rubric_text": "In `slack_data.json`, each channel must have the exact number of messages: #hr-onboarding=15, #hr-offboarding=1, #hr-leave-requests=13, #hr-performance=2, #hr-recruiting=3, #hr-escalations=0, #hr-general=3",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n expected = {\n \"hr-onboarding\": 15,\n \"hr-offboarding\": 1,\n \"hr-leave-requests\": 13,\n \"hr-performance\": 2,\n \"hr-recruiting\": 3,\n \"hr-escalations\": 0,\n \"hr-general\": 3\n }\n name_to_id = {}\n for cid, cdata in channels.items():\n name = cdata.get(\"name\", \"\").lower().strip(\"#\")\n name_to_id[name] = cid\n failures = []\n for channel_name, expected_count in expected.items():\n cid = name_to_id.get(channel_name)\n if cid is None:\n if expected_count == 0:\n continue\n failures.append(f\"#{channel_name} not found (expected {expected_count} messages)\")\n continue\n actual_count = len(messages.get(cid, []))\n if actual_count != expected_count:\n failures.append(f\"#{channel_name}: expected {expected_count} messages, found {actual_count}\")\n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Message count mismatches: \" + \"; \".join(failures)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All channels have the correct number of messages.\"}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "100b7c72-5bd9-4716-93ac-939303f7d470",
|
|||
|
|
"sort_order": 19,
|
|||
|
|
"rubric_text": "In `jira_state.json`, there must be exactly 8 issues",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n if isinstance(data, dict) and \"issues\" in data:\n issues = data[\"issues\"]\n else:\n issues = data\n if isinstance(issues, dict):\n count = len(issues)\n elif isinstance(issues, list):\n count = len(issues)\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Unexpected issues format: {type(issues)}\"}\n if count == 8:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 8 issues in jira_state.json.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 8 issues in jira_state.json, found {count}.\"}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
}
|
|||
|
|
]
|