Add Handbook.md benchmark tasks
This commit is contained in:
commit
72f9a65917
1800 files changed
+323589
No files matched your search
@@ -0,0 +1,142 @@
|
||||
[
|
||||
{
|
||||
"id": "e9e30a9b-fdae-4d1a-b35a-bce846762f57",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `mailbox.json`, no sent emails must exist with `to_addr` set to `facilities@crestwood.edu`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n violations = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\" and \"facilities@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} sent email(s) to facilities@crestwood.edu, expected none.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails to facilities@crestwood.edu found.\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "e7bfa402-fa2a-4fc5-bdf6-b5099fa59d0a",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `mailbox.json`, the email from `facilities@crestwood.edu` must be in the `general` folder",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n from_facilities = [e for e in emails if \"facilities@crestwood.edu\" in str(e.get(\"from_addr\", \"\")).lower()]\n if not from_facilities:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email from facilities@crestwood.edu found.\"}\n for e in from_facilities:\n folder = e.get(\"folder\", \"\").lower()\n if \"general\" not in folder:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email from facilities@crestwood.edu is in folder '{e.get('folder')}', expected 'general'.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email from facilities@crestwood.edu is in the general folder.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e7f91235-98e4-4389-95cf-2b7767dea2f1",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `slack_data.json`, channel `#hr-general` must contain a message with text `[OUT OF SCOPE] - HR office light bulb out – Main Campus rm 214`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n hr_general_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-general\":\n hr_general_id = cid\n break\n if hr_general_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-general not found.\"}\n messages = data.get(\"messages\", {}).get(hr_general_id, [])\n\n def normalize_dashes(s):\n # Collapse any hyphen/en-dash/em-dash/minus to a single canonical '-'\n return (s.replace(\"\\u2014\", \"-\")\n .replace(\"\\u2013\", \"-\")\n .replace(\"\\u2212\", \"-\")\n .replace(\"-\", \"-\"))\n\n target = normalize_dashes(\"[OUT OF SCOPE] - HR office light bulb out - Main Campus rm 214\").lower()\n for msg in messages:\n text = normalize_dashes(msg.get(\"text\", \"\")).lower()\n if target in text:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required message in #hr-general: '{msg.get('text')}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Message '[OUT OF SCOPE] - HR office light bulb out - Main Campus rm 214' (any dash variant) not found in #hr-general. Messages found: {[m.get('text') for m in messages]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "6059ef2b-9c84-4307-8785-3c9e1948af51",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `mailbox.json`, the email from `l.fontaine@crestwood.edu` must be in the `leave` folder",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n from_fontaine = [e for e in emails if \"l.fontaine@crestwood.edu\" in str(e.get(\"from_addr\", \"\")).lower()]\n if not from_fontaine:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email from l.fontaine@crestwood.edu found.\"}\n for e in from_fontaine:\n folder = e.get(\"folder\", \"\").lower()\n if \"leave\" not in folder:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email from l.fontaine@crestwood.edu is in folder '{e.get('folder')}', expected 'leave'.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email from l.fontaine@crestwood.edu is in the leave folder.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "fa6332bb-e59b-433e-b135-335bcb794eef",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `mailbox.json`, a sent email to `l.fontaine@crestwood.edu` must exist with body containing PTO balance of 68 hours, accrual rate of 4.62 hours per pay period, and contact info for Patricia Huang at p.huang@crestwood.edu",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n candidates = [e for e in emails if \"l.fontaine@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to l.fontaine@crestwood.edu found.\"}\n required_phrases = [\"68 hours\", \"4.62\", \"p.huang@crestwood.edu\", \"04/07/2026\"]\n for e in candidates:\n body = e.get(\"body_text\", \"\").lower()\n if all(phrase.lower() in body for phrase in required_phrases):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email to l.fontaine@crestwood.edu found with required PTO balance content.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email to l.fontaine@crestwood.edu found but missing required content. Required: {required_phrases}. Bodies found: {[e.get('body_text','')[:200] for e in candidates]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4ed6f0e7-1d75-45cc-a70d-a53741123f63",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `mailbox.json`, a sent email to `p.huang@crestwood.edu` with CC `t.salazar@crestwood.edu` with subject `I-9 Urgent Reminder — Salazar, CU-1026` must exist with body mentioning Tomás Salazar, CU-1026, 04/08/2026 federal deadline, and Section 2 of I-9",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n\n # Find emails sent to p.huang@crestwood.edu\n candidates = []\n for e in emails:\n to_addr = e.get(\"to_addr\", \"\") or e.get(\"to\", \"\")\n if isinstance(to_addr, list):\n to_lower = [a.lower() for a in to_addr]\n else:\n to_lower = [str(to_addr).lower()]\n if any(\"p.huang@crestwood.edu\" in a for a in to_lower):\n candidates.append(e)\n\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to p.huang@crestwood.edu found.\"}\n\n # Filter for I-9 / Salazar / CU-1026 related subject\n salazar_emails = []\n for e in candidates:\n subj = e.get(\"subject\", \"\").lower()\n if \"salazar\" in subj or \"cu-1026\" in subj or \"i-9\" in subj or \"i9\" in subj:\n salazar_emails.append(e)\n\n if not salazar_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No I-9/Salazar-related email found among emails to p.huang@crestwood.edu. Subjects found: {[e.get('subject') for e in candidates]}\"}\n\n for e in salazar_emails:\n body = e.get(\"body_text\", \"\") or e.get(\"body\", \"\") or \"\"\n subject = e.get(\"subject\", \"\")\n full_text = (subject + \" \" + body).lower()\n\n # Check CC includes t.salazar@crestwood.edu\n cc = e.get(\"cc_addr\", \"\") or e.get(\"cc\", \"\") or \"\"\n if isinstance(cc, list):\n cc_lower = [a.lower() for a in cc]\n else:\n cc_lower = [str(cc).lower()]\n has_cc = any(\"t.salazar@crestwood.edu\" in c for c in cc_lower)\n\n # Check body mentions: Salazar (or Tomás Salazar), CU-1026, April 8 2026 deadline, Section 2\n has_salazar = \"salazar\" in full_text\n has_cu1026 = \"cu-1026\" in full_text or \"cu1026\" in full_text\n\n # Accept various date formats for April 8, 2026\n date_patterns = [\n \"04/08/2026\", \"4/8/2026\",\n ]\n has_date = any(p in full_text for p in date_patterns)\n\n # Section 2 check - be lenient\n has_section2 = \"section 2\" in full_text or \"section ii\" in full_text or \"section two\" in full_text or \"section-2\" in full_text\n\n if has_salazar and has_cu1026 and has_date and has_section2:\n if not has_cc:\n return {\"pass\": True, \"score\": 0.8, \"feedback\": f\"Found I-9 urgent reminder email to p.huang@crestwood.edu for Salazar CU-1026 with required content, but CC to t.salazar@crestwood.edu not confirmed. CC field: {cc}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found I-9 urgent reminder email to p.huang@crestwood.edu with CC t.salazar@crestwood.edu for Salazar CU-1026 with all required content (Tomás Salazar, CU-1026, April 8 2026 deadline, Section 2).\"}\n\n # Provide detailed failure info\n details = []\n for e in salazar_emails:\n body = e.get(\"body_text\", \"\") or e.get(\"body\", \"\") or \"\"\n full_text = (e.get(\"subject\", \"\") + \" \" + body).lower()\n cc = e.get(\"cc_addr\", \"\") or e.get(\"cc\", \"\") or \"\"\n missing = []\n if \"salazar\" not in full_text:\n missing.append(\"Salazar\")\n if \"cu-1026\" not in full_text and \"cu1026\" not in full_text:\n missing.append(\"CU-1026\")\n date_patterns = [\n \"april 8\", \"april 08\", \"04/08/2026\", \"4/8/2026\",\n \"2026-04-08\", \"apr 8\", \"apr 08\", \"april 8, 2026\",\n \"april 8th\", \"april 8th, 2026\", \"04-08-2026\",\n \"4-8-2026\", \"april 08, 2026\"\n ]\n if not any(p in full_text for p in date_patterns):\n missing.append(\"April 8 2026 date\")\n if \"section 2\" not in full_text and \"section ii\" not in full_text and \"section two\" not in full_text and \"section-2\" not in full_text:\n missing.append(\"Section 2\")\n details.append({\"subject\": e.get(\"subject\"), \"cc\": cc, \"missing\": missing, \"body_snippet\": full_text[:300]})\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"I-9 urgent reminder email found but missing required content. Details: {details}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "559efc43-ae89-4956-8045-90ca9cb46fbb",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `benefits_enrollment.xlsx`, cell C26 must be set to `Default Enrollment` and cell D26 must be set to `05/01/2026`",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / \"benefits_enrollment.xlsx\"\n if not wb_path.exists():\n # Try searching\n matches = list(Path(workspace_path).glob(\"**/benefits_enrollment.xlsx\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"benefits_enrollment.xlsx not found in workspace.\"}\n wb_path = matches[0]\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n c26 = ws[\"C26\"].value\n d26 = ws[\"D26\"].value\n c26_str = str(c26).strip() if c26 is not None else \"\"\n d26_str = str(d26).strip() if d26 is not None else \"\"\n c26_ok = \"default enrollment\" in c26_str.lower()\n d26_ok = \"05/01/2026\" in d26_str or d26_str == \"05/01/2026\"\n # Also check if d26 is a date object\n import datetime\n if hasattr(d26, 'month') and hasattr(d26, 'year'):\n d26_ok = d26.month == 5 and d26.year == 2026 and d26.day == 1\n if c26_ok and d26_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"C26='{c26}' (Default Enrollment) and D26='{d26}' (05/01/2026) correctly set.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"C26='{c26}' (expected 'Default Enrollment', ok={c26_ok}), D26='{d26}' (expected 05/01/2026, ok={d26_ok}).\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4323ac49-9f39-4121-9b6d-13ab81d30ed0",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `onboarding_checklist.xlsx`, cell E15 must be set to `05/01/2026` (Benefits Enrolled date)",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / \"onboarding_checklist.xlsx\"\n if not wb_path.exists():\n matches = list(Path(workspace_path).glob(\"**/onboarding_checklist.xlsx\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"onboarding_checklist.xlsx not found in workspace.\"}\n wb_path = matches[0]\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n e15 = ws[\"E15\"].value\n e15_str = str(e15).strip() if e15 is not None else \"\"\n ok = \"05/01/2026\" in e15_str\n if hasattr(e15, 'month') and hasattr(e15, 'year'):\n ok = e15.month == 5 and e15.year == 2026 and e15.day == 1\n if ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"E15='{e15}' correctly set to 05/01/2026 for Benefits Enrolled.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"E15='{e15}' does not match expected 05/01/2026.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "f2b2962e-5acc-4346-a75d-c52075fa9e35",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `employee_roster.xlsx`, cell L23 must be set to `Active`",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n wb_path = Path(workspace_path) / \"employee_roster.xlsx\"\n if not wb_path.exists():\n matches = list(Path(workspace_path).glob(\"**/employee_roster.xlsx\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"employee_roster.xlsx not found in workspace.\"}\n wb_path = matches[0]\n wb = openpyxl.load_workbook(wb_path)\n ws = wb.active\n l23 = ws[\"L23\"].value\n l23_str = str(l23).strip() if l23 is not None else \"\"\n if l23_str.lower() == \"active\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"L23='{l23}' correctly set to Active.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"L23='{l23}' does not match expected 'Active'.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "efd47f2e-f94d-4e44-b54b-7b67ad1ba7fe",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `slack_data.json`, channel `#hr-onboarding` must contain a message with exact text `ONBOARDING COMPLETE: William Trent, CU-1022, 04/07/2026`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n hr_onboarding_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-onboarding\":\n hr_onboarding_id = cid\n break\n if hr_onboarding_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-onboarding not found.\"}\n messages = data.get(\"messages\", {}).get(hr_onboarding_id, [])\n target = \"ONBOARDING COMPLETE: William Trent, CU-1022, 04/07/2026\"\n for msg in messages:\n if target.lower() in msg.get(\"text\", \"\").lower():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required message in #hr-onboarding: '{msg.get('text')}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Message '{target}' not found in #hr-onboarding. Messages: {[m.get('text') for m in messages]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "62bb6384-ef1c-4030-b4d3-3dd3ff76d725",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In `mailbox.json`, a sent email to `w.trent@crestwood.edu` with subject `Default Benefits Enrollment Confirmation — Trent, CU-1022` must exist with body mentioning default enrollment, 04/02/2026 deadline, and 05/01/2026 effective date",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n candidates = [e for e in emails if \"w.trent@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to w.trent@crestwood.edu found.\"}\n subject_match = [e for e in candidates if \"cu-1022\" in e.get(\"subject\", \"\").lower() or \"trent\" in e.get(\"subject\", \"\").lower()]\n required_phrases = [\"04/02/2026\", \"05/01/2026\", \"default\"]\n for e in subject_match:\n body = e.get(\"body_text\", \"\")\n if all(phrase.lower() in body.lower() for phrase in required_phrases):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found default benefits enrollment confirmation email to w.trent@crestwood.edu with required content.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Default benefits enrollment confirmation email to w.trent@crestwood.edu not found or missing required content. Required: {required_phrases}. Subjects: {[e.get('subject') for e in candidates]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "97c5f3d5-b336-4a10-b782-d15d710b62d8",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "In `mailbox.json`, a sent email to `p.huang@crestwood.edu` with subject `FMLA Deadline Passed — CU-1023` must exist with body mentioning Meghan Cross FMLA Certification overdue",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n candidates = [e for e in emails if \"p.huang@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to p.huang@crestwood.edu found.\"}\n fmla_emails = [e for e in candidates if \"fmla\" in e.get(\"subject\", \"\").lower() and \"cu-1023\" in e.get(\"subject\", \"\").lower()]\n if not fmla_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No FMLA CU-1023 email to p.huang@crestwood.edu found. Subjects: {[e.get('subject') for e in candidates]}\"}\n for e in fmla_emails:\n body = e.get(\"body_text\", \"\")\n if \"cross\" in body.lower() or \"meghan\" in body.lower():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found FMLA Deadline Passed email to p.huang@crestwood.edu for CU-1023 (Meghan Cross).\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FMLA CU-1023 email to p.huang@crestwood.edu found but body missing 'Cross'/'Meghan'. Bodies: {[e.get('body_text','')[:200] for e in fmla_emails]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "342775db-78be-4cd8-a801-e523a0286686",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In `mailbox.json`, a sent email to `m.cross@crestwood.edu` with subject `FMLA Certification Overdue — Cross, CU-1023` OR `Template 18 — Cross, CU-1023` OR `Template 18 – FMLA Certification Overdue — Cross, CU-1023` must exist with body mentioning due date 04/07/2026, FMLA Medical Certification Form, and contact info for Patricia Huang",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n # Find emails sent to m.cross@crestwood.edu\n candidates = [e for e in emails if \"m.cross@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to m.cross@crestwood.edu found.\"}\n\n # Check subject matches one of the allowed patterns (loose matching)\n subject_keywords = [\"fmla\", \"template 18\", \"cross\", \"cu-1023\"]\n\n # Check body for required content\n # Due date: 04/07/2026 in various formats\n date_patterns = [\"04/07/2026\", \"4/7/2026\"]\n # FMLA Medical Certification Form\n fmla_form_patterns = [\"fmla medical certification\", \"fmla certification\", \"medical certification form\"]\n # Contact info for Patricia Huang - could be email, phone, or just name\n patricia_patterns = [\"patricia huang\", \"p.huang@crestwood.edu\", \"p. huang\"]\n\n for e in candidates:\n body = (e.get(\"body_text\", \"\") or \"\").lower()\n subject = (e.get(\"subject\", \"\") or \"\").lower()\n combined = subject + \" \" + body\n\n # Subject check: must relate to FMLA or Template 18 and Cross/CU-1023\n subject_ok = False\n if (\"fmla\" in subject or \"template 18\" in subject) and (\"cross\" in subject or \"cu-1023\" in subject):\n subject_ok = True\n\n if not subject_ok:\n continue\n\n # Date check\n date_ok = any(dp in combined for dp in date_patterns)\n\n # FMLA form check\n fmla_form_ok = any(fp in combined for fp in fmla_form_patterns)\n\n # Patricia Huang contact info check\n patricia_ok = any(pp in combined for pp in patricia_patterns)\n\n if date_ok and fmla_form_ok and patricia_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found FMLA Certification Overdue email to m.cross@crestwood.edu with required content. Subject: {e.get('subject')}\"}\n\n # Collect what's missing for feedback\n missing = []\n if not date_ok:\n missing.append(\"due date (04/07/2026)\")\n if not fmla_form_ok:\n missing.append(\"FMLA Medical Certification Form reference\")\n if not patricia_ok:\n missing.append(\"Patricia Huang contact info\")\n\n # If we get here, no email matched fully\n all_subjects = [e.get('subject', '') for e in candidates]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FMLA Certification Overdue email to m.cross@crestwood.edu not found or missing required content. Need: due date 04/07/2026, FMLA Medical Certification Form mention, and Patricia Huang contact info. Candidate subjects: {all_subjects}\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "543a04a0-b8fa-4f83-96f1-70823791a25d",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `slack_data.json`, channel `#hr-leave-requests` must contain a message with text `FMLA CERTIFICATION OVERDUE: Meghan Cross, CU-1023, Due: 04/07/2026` and mentioning overdue",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n hr_leave_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-leave-requests\":\n hr_leave_id = cid\n break\n if hr_leave_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-leave-requests not found.\"}\n messages = data.get(\"messages\", {}).get(hr_leave_id, [])\n required_phrases = [\"FMLA CERTIFICATION OVERDUE\", \"Meghan Cross\", \"CU-1023\", \"04/07/2026\"]\n for msg in messages:\n text = msg.get(\"text\", \"\")\n if all(phrase.lower() in text.lower() for phrase in required_phrases):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required FMLA overdue message in #hr-leave-requests: '{text}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Required FMLA overdue message not found in #hr-leave-requests. Required phrases: {required_phrases}. Messages: {[m.get('text') for m in messages]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "6b40093b-ab7a-46b8-9129-bcd2c58d223a",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "In `jira_state.json`, issue `HR-7` must have comments containing all required entries for steps 11.3 and 11.4, and description appended with `Status — CLOSED`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n def adf_to_text(node):\n \"\"\"Flatten an Atlassian Document Format node (dict/list) to plain text. Passthrough for strings.\"\"\"\n if node is None:\n return \"\"\n if isinstance(node, str):\n return node\n out = []\n def walk(n):\n if isinstance(n, dict):\n if n.get(\"type\") == \"text\":\n out.append(n.get(\"text\", \"\"))\n for c in n.get(\"content\", []) or []:\n walk(c)\n if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n out.append(\"\\n\")\n elif isinstance(n, list):\n for x in n:\n walk(x)\n walk(node)\n return \"\".join(out).strip()\n\n # Navigate to issues\n issues = data.get(\"issues\", data) if isinstance(data, dict) else data\n\n trent_issue = None\n matched_key = None\n if isinstance(issues, dict):\n for key, val in issues.items():\n if \"trent\" in key.lower() and \"onboard\" in key.lower():\n trent_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n if trent_issue is None:\n for key, val in issues.items():\n combined = (key + \" \" + json.dumps(val, default=str)).lower()\n if \"trent\" in combined and \"onboard\" in combined:\n trent_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict):\n item_str = json.dumps(item, default=str).lower()\n if \"trent\" in item_str and \"onboard\" in item_str:\n trent_issue = item\n matched_key = item.get(\"key\", \"list_item\")\n break\n\n if trent_issue is None:\n all_keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A (list)\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-7 not found in jira_state.json. Available keys: {all_keys}\"}\n\n # Gather comments. Handle both schemas:\n # (a) comments nested inside the issue: issue[\"comments\"] = [...]\n # (b) comments stored in a parallel top-level dict: data[\"comments\"][issue_key] = [...]\n # Each comment's body may be a plain string OR an ADF dict.\n comments_raw = []\n nested = trent_issue.get(\"comments\")\n if isinstance(nested, list):\n comments_raw.extend(nested)\n if isinstance(data, dict):\n parallel = data.get(\"comments\", {})\n if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n comments_raw.extend(parallel[matched_key])\n\n comments_text_parts = []\n for c in comments_raw:\n if isinstance(c, dict):\n for field in (\"body\", \"text\", \"content\"):\n if field in c:\n comments_text_parts.append(adf_to_text(c[field]))\n else:\n comments_text_parts.append(str(c))\n comments_text = \" \".join(comments_text_parts)\n comments_text_lower = comments_text.lower()\n\n # Description. Handle both schemas:\n # (a) issue[\"description\"] directly (string or ADF)\n # (b) issue[\"fields\"][\"description\"] (string or ADF)\n description = trent_issue.get(\"description\")\n if description in (None, \"\") and isinstance(trent_issue.get(\"fields\"), dict):\n description = trent_issue[\"fields\"].get(\"description\")\n description_text = adf_to_text(description)\n desc_lower = description_text.lower()\n\n all_text_lower = comments_text_lower + \" \" + desc_lower\n\n required_comment_fragments = [\n (\"11.3\", [\"11.3\", \"step 11.3\", \"11-3\"]),\n (\"benefits_enrollment\", [\"benefits_enrollment\", \"benefits enrollment\", \"benefits-enrollment\"]),\n (\"Default Enrollment\", [\"default enrollment\", \"default enroll\"]),\n (\"05/01/2026\", [\"05/01/2026\", \"5/1/2026\"]),\n (\"Template 10\", [\"template 10\", \"template-10\", \"template_10\"]),\n (\"w.trent@crestwood.edu\", [\"w.trent@crestwood\", \"trent@crestwood\"]),\n (\"11.4\", [\"11.4\", \"step 11.4\", \"11-4\"]),\nLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "fd6fcf96-a025-43ef-b1ec-d6bd8023ec7d",
|
||||
"sort_order": 15,
|
||||
"rubric_text": "In `jira_state.json`, issue `HR-1` must have comments for Template 18 sent to m.cross@crestwood.edu, slack posted to #hr-leave-requests, email forwarded to p.huang@crestwood.edu, and waiting on Patricia Huang",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n def adf_to_text(node):\n if node is None:\n return \"\"\n if isinstance(node, str):\n return node\n out = []\n def walk(n):\n if isinstance(n, dict):\n if n.get(\"type\") == \"text\":\n out.append(n.get(\"text\", \"\"))\n for c in n.get(\"content\", []) or []:\n walk(c)\n if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n out.append(\"\\n\")\n elif isinstance(n, list):\n for x in n:\n walk(x)\n walk(node)\n return \"\".join(out).strip()\n\n issues = data.get(\"issues\", data) if isinstance(data, dict) and \"issues\" in data else data\n\n cross_issue = None\n matched_key = None\n if isinstance(issues, dict):\n for key, val in issues.items():\n if \"cross\" in key.lower() and \"leave\" in key.lower():\n cross_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict):\n s = json.dumps(item, default=str).lower()\n if \"cross\" in s and \"leave\" in s:\n cross_issue = item\n matched_key = item.get(\"key\", \"list_item\")\n break\n\n if cross_issue is None:\n keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-1 not found. Keys: {keys}\"}\n\n comments_raw = []\n nested = cross_issue.get(\"comments\")\n if isinstance(nested, list):\n comments_raw.extend(nested)\n if isinstance(data, dict):\n parallel = data.get(\"comments\", {})\n if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n comments_raw.extend(parallel[matched_key])\n\n parts = []\n for c in comments_raw:\n if isinstance(c, dict):\n for field in (\"body\", \"text\", \"content\"):\n if field in c:\n parts.append(adf_to_text(c[field]))\n else:\n parts.append(str(c))\n comments_text = \" \".join(parts)\n ct = comments_text.lower()\n\n required_fragments = [\"template 18\", \"m.cross@crestwood.edu\", \"hr-leave-requests\", \"p.huang@crestwood.edu\"]\n missing = [f for f in required_fragments if f.lower() not in ct]\n if missing:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"HR-1 (key: {matched_key}) missing fragments: {missing}. Found {len(comments_raw)} comments. Text: '{comments_text[:400]}'\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"HR-1 (key: {matched_key}) has required fragments across {len(comments_raw)} comments.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "0b596b96-db7e-445f-a83b-05ef87e66771",
|
||||
"sort_order": 16,
|
||||
"rubric_text": "In `jira_state.json`, issue `HR-6` must have a comment mentioning Template 6 sent to p.huang@crestwood.edu and step 11.2",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n def adf_to_text(node):\n \"\"\"Flatten Atlassian Document Format (dict/list) to plain text; passthrough for strings.\"\"\"\n if node is None:\n return \"\"\n if isinstance(node, str):\n return node\n out = []\n def walk(n):\n if isinstance(n, dict):\n if n.get(\"type\") == \"text\":\n out.append(n.get(\"text\", \"\"))\n for c in n.get(\"content\", []) or []:\n walk(c)\n if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n out.append(\"\\n\")\n elif isinstance(n, list):\n for x in n:\n walk(x)\n walk(node)\n return \"\".join(out).strip()\n\n issues = data.get(\"issues\", data) if isinstance(data, dict) and \"issues\" in data else data\n\n salazar_issue = None\n matched_key = None\n if isinstance(issues, dict):\n for key, val in issues.items():\n if \"salazar\" in key.lower() and \"onboard\" in key.lower():\n salazar_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n if salazar_issue is None:\n for key, val in issues.items():\n combined = (key + \" \" + json.dumps(val, default=str)).lower()\n if \"salazar\" in combined and \"onboard\" in combined:\n salazar_issue = val if isinstance(val, dict) else {\"raw\": val}\n matched_key = key\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict):\n s = json.dumps(item, default=str).lower()\n if \"salazar\" in s and \"onboard\" in s:\n salazar_issue = item\n matched_key = item.get(\"key\", \"list_item\")\n break\n\n if salazar_issue is None:\n keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-6 not found. Keys: {keys}\"}\n\n # Collect comments from both possible schemas.\n comments_raw = []\n nested = salazar_issue.get(\"comments\")\n if isinstance(nested, list):\n comments_raw.extend(nested)\n if isinstance(data, dict):\n parallel = data.get(\"comments\", {})\n if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n comments_raw.extend(parallel[matched_key])\n\n parts = []\n for c in comments_raw:\n if isinstance(c, dict):\n for field in (\"body\", \"text\", \"content\"):\n if field in c:\n parts.append(adf_to_text(c[field]))\n else:\n parts.append(str(c))\n comments_text = \" \".join(parts)\n ct = comments_text.lower()\n\n required_fragments = [\"11.2\", \"Template 6\", \"p.huang@crestwood.edu\"]\n missing = [f for f in required_fragments if f.lower() not in ct]\n if missing:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"HR-6 (key: {matched_key}) missing comment fragments: {missing}. Found {len(comments_raw)} comments. Text: '{comments_text[:400]}'\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"HR-6 (key: {matched_key}) has required comment for Template 6 to p.huang@crestwood.edu across {len(comments_raw)} entries.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "0968f22e-9afc-4fa9-8baf-f8ed52566e11",
|
||||
"sort_order": 17,
|
||||
"rubric_text": "In `mailbox.json`, there must be exactly 11 sent emails (emails with folder='Sent')",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n if len(sent) == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 11 sent emails. Subjects: {[e.get('subject') for e in sent]}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 11 sent emails, found {len(sent)}. Subjects: {[e.get('subject') for e in sent]}\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "9fa6493a-d029-4e8f-a6ab-fdade61b8c30",
|
||||
"sort_order": 18,
|
||||
"rubric_text": "In `slack_data.json`, each channel must have the exact number of messages: #hr-onboarding=15, #hr-offboarding=1, #hr-leave-requests=13, #hr-performance=2, #hr-recruiting=3, #hr-escalations=0, #hr-general=3",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n expected = {\n \"hr-onboarding\": 15,\n \"hr-offboarding\": 1,\n \"hr-leave-requests\": 13,\n \"hr-performance\": 2,\n \"hr-recruiting\": 3,\n \"hr-escalations\": 0,\n \"hr-general\": 3\n }\n name_to_id = {}\n for cid, cdata in channels.items():\n name = cdata.get(\"name\", \"\").lower().strip(\"#\")\n name_to_id[name] = cid\n failures = []\n for channel_name, expected_count in expected.items():\n cid = name_to_id.get(channel_name)\n if cid is None:\n if expected_count == 0:\n continue\n failures.append(f\"#{channel_name} not found (expected {expected_count} messages)\")\n continue\n actual_count = len(messages.get(cid, []))\n if actual_count != expected_count:\n failures.append(f\"#{channel_name}: expected {expected_count} messages, found {actual_count}\")\n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Message count mismatches: \" + \"; \".join(failures)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All channels have the correct number of messages.\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "100b7c72-5bd9-4716-93ac-939303f7d470",
|
||||
"sort_order": 19,
|
||||
"rubric_text": "In `jira_state.json`, there must be exactly 8 issues",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n if isinstance(data, dict) and \"issues\" in data:\n issues = data[\"issues\"]\n else:\n issues = data\n if isinstance(issues, dict):\n count = len(issues)\n elif isinstance(issues, list):\n count = len(issues)\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Unexpected issues format: {type(issues)}\"}\n if count == 8:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 8 issues in jira_state.json.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 8 issues in jira_state.json, found {count}.\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Today is April 7, 2026, and it's already noon. i've been out a few days need you to go through my inbox and handle every unread message. Also, for any items for which an SOP deadline has passed, I need you to update relevant documents, send out necessary emails and slack messages, update Jira tickets, and make any other changes as necessary. Advance deadline-passed items as far as possible. If a Jira ticket indicates that an action has been taken, you should trust the Jira ticket. I've uploaded our SOP which you will need to follow, and all relevant employee files.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user