DerekSurge
129 lines
58 KiB
JSON
129 lines
58 KiB
JSON
[
|
||
{
|
||
"id": "cb20ffd5-5d81-4814-ae29-03c32e1ff398",
|
||
"sort_order": 0,
|
||
"rubric_text": "In `mailbox.json`, no sent email with `from_addr == hr@crestwood.edu` must have a subject containing 'Template' (case-insensitive).",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; skipping check.\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n violations = []\n for email in data.get(\"emails\", []):\n if email.get(\"folder\", \"\").lower() == \"sent\" and email.get(\"from_addr\", \"\").lower() == \"hr@crestwood.edu\":\n subject = email.get(\"subject\", \"\")\n if \"template\" in subject.lower():\n violations.append(subject)\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found sent emails from hr@crestwood.edu with 'Template' in subject: {violations}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails from hr@crestwood.edu have 'Template' in subject.\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "a320280d-d6c5-4e61-a78f-20d77a54127d",
|
||
"sort_order": 1,
|
||
"rubric_text": "In `mailbox.json`, exactly one sent email must be addressed to m.torres@crestwood.edu, and exactly one sent email must exist addressed to p.huang@crestwood.edu.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; skipping check.\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n torres_emails = [e for e in sent_emails if \"m.torres@crestwood.edu\" in e.get(\"to_addr\", \"\").lower()]\n huang_emails = [e for e in sent_emails if \"p.huang@crestwood.edu\" in e.get(\"to_addr\", \"\").lower()]\n issues = []\n if len(torres_emails) != 1:\n issues.append(f\"Expected exactly 1 sent email to m.torres@crestwood.edu, found {len(torres_emails)}\")\n if len(huang_emails) != 1:\n issues.append(f\"Expected exactly 1 sent email to p.huang@crestwood.edu, found {len(huang_emails)}\")\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 1 email to m.torres and exactly 1 email to p.huang.\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "8e6b5aaa-6946-428a-a874-eee5e12ea7d5",
|
||
"sort_order": 2,
|
||
"rubric_text": "In jira_state.json, exactly one Jira issue must exist with summary LEAVE-Torres-04122026 in the LEAVE project, and its description must contain all three of: Employee Name: Maria Torres, Employee ID: E-1001, and Assignee: Patricia Huang.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external services path provided; cannot check jira_state.json.\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found at expected path.\"}\n with open(jira_path, encoding=\"utf-8\") as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n all_issues = list(issues.values()) if isinstance(issues, dict) else issues\n\n def get_summary(i):\n return (i.get(\"fields\", {}).get(\"summary\") or i.get(\"summary\") or \"\").strip()\n\n def get_project_key(i):\n p = i.get(\"fields\", {}).get(\"project\") or i.get(\"project\") or \"\"\n if isinstance(p, dict):\n return (p.get(\"key\") or p.get(\"name\") or \"\").upper().strip()\n return str(p).upper().strip()\n\n def extract_description_text(i):\n desc = i.get(\"fields\", {}).get(\"description\") or i.get(\"description\") or {}\n if isinstance(desc, dict):\n try:\n return desc[\"content\"][0][\"content\"][0][\"text\"]\n except (KeyError, IndexError, TypeError):\n return str(desc)\n return str(desc)\n\n target_summary = \"LEAVE-Torres-04122026\"\n matched = [i for i in all_issues if get_summary(i) == target_summary]\n\n if len(matched) == 0:\n all_summaries = [get_summary(i) for i in all_issues]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No issue found with summary '{target_summary}'. All summaries: {all_summaries}\"}\n if len(matched) > 1:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 issue with summary '{target_summary}', found {len(matched)}.\"}\n\n issue = matched[0]\n proj = get_project_key(issue)\n if \"LEAVE\" not in proj:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue found but project is '{proj}', expected 'LEAVE'.\"}\n\n desc_text = extract_description_text(issue)\n required = [\n \"Employee Name: Maria Torres\",\n \"Employee ID: E-1001\",\n \"Assignee: Patricia Huang\",\n ]\n missing = [term for term in required if term.lower() not in desc_text.lower()]\n if missing:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue '{target_summary}' found in LEAVE project but description is missing: {missing}. Description: '{desc_text[:300]}'\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Issue '{target_summary}' found in LEAVE project with all required description fields.\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "dfaba2c5-5d8f-47e5-a647-06aba1f84105",
|
||
"sort_order": 3,
|
||
"rubric_text": "In `slack_data.json`, the `#hr-leave-requests` channel must contain a message with ALL of: 'FMLA REQUEST', 'Maria Torres', 'E-1001', 'Eligible: YES', 'Certification Due:', '04/27/2026'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external services path provided; cannot check slack_data.json.\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at expected path.\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n target_channel_id = None\n for cid, cinfo in channels.items():\n if cinfo.get(\"name\", \"\").lower() == \"hr-leave-requests\":\n target_channel_id = cid\n break\n if target_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-leave-requests not found in slack_data.json\"}\n messages = data.get(\"messages\", {}).get(target_channel_id, [])\n required_terms = [\"FMLA REQUEST\", \"Maria Torres\", \"E-1001\", \"Eligible: YES\", \"Certification Due:\", \"04/27/2026\"]\n for msg in messages:\n text = msg.get(\"text\", \"\")\n if all(term in text for term in required_terms):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found matching message in #hr-leave-requests with all required terms: {required_terms}\"}\n # Provide detailed feedback on what was found\n if not messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No messages found in #hr-leave-requests. Expected a message containing all of: {required_terms}\"}\n # Check partial matches for better feedback\n best_match = 0\n best_text = \"\"\n for msg in messages:\n text = msg.get(\"text\", \"\")\n match_count = sum(1 for term in required_terms if term in text)\n if match_count > best_match:\n best_match = match_count\n best_text = text[:200]\n missing_in_best = [term for term in required_terms if term not in best_text]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in #hr-leave-requests contains all required terms {required_terms}. Best partial match had {best_match}/{len(required_terms)} terms. Missing terms in best match: {missing_in_best}. Best match text preview: '{best_text}'\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "12ef90e8-0558-4d47-9e4b-3b2b014c36e6",
|
||
"sort_order": 4,
|
||
"rubric_text": "In `calendar_data.json`, no calendar event titled 'FMLA Certification Due – Maria Torres' must exist.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping check.\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"calendar_data.json not found; skipping check.\"}\n with open(cal_path) as f:\n data = json.load(f)\n events = data.get(\"events\", {})\n target = \"FMLA Certification Due\"\n torres = \"Maria Torres\"\n found = []\n for eid, ev in events.items():\n summary = ev.get(\"summary\", \"\")\n if target.lower() in summary.lower() and torres.lower() in summary.lower():\n found.append(summary)\n if found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found calendar events that should not exist: {found}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No 'FMLA Certification Due – Maria Torres' calendar event found.\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "b0929e16-e343-4415-910f-ed413a7925be",
|
||
"sort_order": 5,
|
||
"rubric_text": "In `leave_tracker.xlsx` Sheet1, exactly one row for Maria Torres (E-1001) must exist with status containing both 'Approved' and 'Pending Certification Review' (but NOT 'Denied'), leave_start_date representing 06/29/2026, and leave_end_date representing 08/07/2026.",
|
||
"verifier_code": "from pathlib import Path\nfrom datetime import datetime\nimport openpyxl\n\n\ndef _parse_mdy(val):\n if val is None:\n return None\n if isinstance(val, datetime):\n return val.date()\n try:\n return datetime.strptime(str(val).strip(), \"%m/%d/%Y\").date()\n except ValueError:\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n files = list(Path(workspace_path).glob(\"**/leave_tracker.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"leave_tracker.xlsx not found in workspace.\"}\n wb = openpyxl.load_workbook(files[0], data_only=True)\n if \"Sheet1\" not in wb.sheetnames:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Sheet1 not found in leave_tracker.xlsx.\"}\n ws = wb[\"Sheet1\"]\n headers = [str(cell.value).strip().lower() if cell.value else \"\" for cell in ws[1]]\n\n def col(name):\n for i, h in enumerate(headers):\n if name.lower() in h:\n return i\n return -1\n\n name_col = col(\"name\")\n id_col = col(\"id\")\n status_col = col(\"status\")\n start_col = col(\"start\")\n end_col = col(\"end\")\n\n torres_rows = []\n for row in ws.iter_rows(min_row=2, values_only=True):\n name_val = str(row[name_col]).lower() if name_col >= 0 and row[name_col] else \"\"\n id_val = str(row[id_col]).lower() if id_col >= 0 and row[id_col] else \"\"\n if \"torres\" in name_val and \"e-1001\" in id_val:\n torres_rows.append(row)\n if len(torres_rows) != 1:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 row for Maria Torres (E-1001), found {len(torres_rows)}.\"}\n\n row = torres_rows[0]\n status = str(row[status_col]) if status_col >= 0 and row[status_col] else \"\"\n issues = []\n if \"approved\" not in status.lower():\n issues.append(f\"Status '{status}' does not contain 'Approved'\")\n if \"pending certification review\" not in status.lower():\n issues.append(f\"Status '{status}' does not contain 'Pending Certification Review'\")\n if \"denied\" in status.lower():\n issues.append(f\"Status '{status}' contains 'Denied' which is not allowed\")\n\n start_val = _parse_mdy(row[start_col]) if start_col >= 0 else None\n end_val = _parse_mdy(row[end_col]) if end_col >= 0 else None\n expected_start = datetime(2026, 6, 29).date()\n expected_end = datetime(2026, 8, 7).date()\n if start_val != expected_start:\n issues.append(f\"leave_start_date is '{row[start_col] if start_col >= 0 else 'N/A'}', expected MM/DD/YYYY 06/29/2026\")\n if end_val != expected_end:\n issues.append(f\"leave_end_date is '{row[end_col] if end_col >= 0 else 'N/A'}', expected MM/DD/YYYY 08/07/2026\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Maria Torres (E-1001) row verified: status, MM/DD/YYYY dates correct.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "83a1a28b-8460-4566-97c0-033153f0131b",
|
||
"sort_order": 6,
|
||
"rubric_text": "In `mailbox.json`, no sent email must have a subject containing both 'FMLA' and 'Denied'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; skipping check.\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n violations = []\n for email in data.get(\"emails\", []):\n if email.get(\"folder\", \"\").lower() == \"sent\":\n subject = email.get(\"subject\", \"\")\n if \"fmla\" in subject.lower() and \"denied\" in subject.lower():\n violations.append(subject)\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found sent emails with both 'FMLA' and 'Denied' in subject: {violations}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails with both 'FMLA' and 'Denied' in subject.\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "820736f9-d420-4aa6-8739-8a7e545b2481",
|
||
"sort_order": 7,
|
||
"rubric_text": "In `mailbox.json`, email with id=1 must have `is_read == true` and `folder == 'Leave'`.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external services path provided; cannot check mailbox.json.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at expected path.\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n email = next((e for e in emails if str(e.get(\"email_id\", \"\")) == \"1\" or e.get(\"email_id\") == 1), None)\n if email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email with id=1 not found in mailbox.json.\"}\n issues = []\n is_read = email.get(\"is_read\", False)\n if isinstance(is_read, str):\n is_read = is_read.lower() in (\"true\", \"1\", \"yes\")\n if not is_read:\n issues.append(\"is_read is not True (actual: {})\".format(email.get(\"is_read\")))\n folder = str(email.get(\"folder\", \"\")).strip()\n if folder.lower() != \"leave\":\n issues.append(\"folder is '{}', expected 'Leave'\".format(folder))\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email id=1: {}\".format(\"; \".join(issues))}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email id=1 is_read=True and folder='{}'.\".format(folder)}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "edb9df50-27d1-4e2a-8835-bf90108fb6d1",
|
||
"sort_order": 8,
|
||
"rubric_text": "Exactly one Jira issue must exist with summary containing 'OFFBOARD-Kim'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external services path provided; cannot check Jira state.\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found; cannot verify Jira issues.\"}\n with open(jira_path) as f:\n data = json.load(f)\n issues = data.get(\"issues\", {})\n all_issues = list(issues.values()) if isinstance(issues, dict) else issues\n\n def get_summary(i):\n return i.get(\"fields\", {}).get(\"summary\") or i.get(\"summary\") or \"\"\n\n all_summaries = [get_summary(i) for i in all_issues]\n\n # Criterion: Exactly one issue with summary containing 'OFFBOARD-Kim' (case-insensitive, flexible separators)\n kim_issues = [i for i in all_issues if \"offboard-kim\" in get_summary(i).lower() or \"offboard kim\" in get_summary(i).lower() or \"offboard_kim\" in get_summary(i).lower()]\n if len(kim_issues) == 0:\n # Also try a more lenient match: summary contains both 'offboard' and 'kim'\n kim_issues = [i for i in all_issues if \"offboard\" in get_summary(i).lower() and \"kim\" in get_summary(i).lower()]\n\n if len(kim_issues) != 1:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 1 issue with summary containing 'OFFBOARD-Kim', found {len(kim_issues)}. All summaries: {all_summaries}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Exactly 1 OFFBOARD-Kim issue found (summary: '{get_summary(kim_issues[0])}'). All summaries: {all_summaries}\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "263bdfe0-972e-4147-afca-5cc43ec39846",
|
||
"sort_order": 9,
|
||
"rubric_text": "In `employee_roster.xlsx` Sheet1, the row for David Kim (E-1002) must have status == 'Separated', last_day representing 04/06/2026, and separation_type == 'Voluntary Resignation'.",
|
||
"verifier_code": "from pathlib import Path\nfrom datetime import datetime\nimport openpyxl\n\n\ndef _parse_mdy(val):\n if val is None:\n return None\n if isinstance(val, datetime):\n return val.date()\n try:\n return datetime.strptime(str(val).strip(), \"%m/%d/%Y\").date()\n except ValueError:\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n files = list(Path(workspace_path).glob(\"**/employee_roster.xlsx\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"employee_roster.xlsx not found in workspace.\"}\n wb = openpyxl.load_workbook(files[0], data_only=True)\n if \"Sheet1\" not in wb.sheetnames:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Sheet1 not found in employee_roster.xlsx.\"}\n ws = wb[\"Sheet1\"]\n headers = [str(cell.value).strip().lower() if cell.value else \"\" for cell in ws[1]]\n\n def col(name):\n for i, h in enumerate(headers):\n if name.lower() in h:\n return i\n return -1\n\n first_col = col(\"first_name\")\n last_col = col(\"last_name\")\n id_col = col(\"id\")\n status_col = col(\"status\")\n last_day_col = col(\"last_day\")\n sep_type_col = col(\"separation_type\")\n\n kim_rows = []\n for row in ws.iter_rows(min_row=2, values_only=True):\n fn = str(row[first_col]).lower() if first_col >= 0 and row[first_col] else \"\"\n ln = str(row[last_col]).lower() if last_col >= 0 and row[last_col] else \"\"\n id_val = str(row[id_col]).lower() if id_col >= 0 and row[id_col] else \"\"\n if ln == \"kim\" and fn == \"david\" and \"e-1002\" in id_val:\n kim_rows.append(row)\n if not kim_rows:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No row found for David Kim (E-1002) in employee_roster.xlsx Sheet1.\"}\n\n row = kim_rows[0]\n issues = []\n status = str(row[status_col]).strip() if status_col >= 0 and row[status_col] else \"\"\n if status.lower() != \"separated\":\n issues.append(f\"status is '{status}', expected 'Separated'\")\n\n last_day = _parse_mdy(row[last_day_col]) if last_day_col >= 0 else None\n expected_last_day = datetime(2026, 4, 6).date()\n if last_day != expected_last_day:\n issues.append(f\"last_day is '{row[last_day_col] if last_day_col >= 0 else 'N/A'}', expected MM/DD/YYYY 04/06/2026\")\n\n sep_type = str(row[sep_type_col]).strip() if sep_type_col >= 0 and row[sep_type_col] else \"\"\n if \"voluntary resignation\" not in sep_type.lower():\n issues.append(f\"separation_type is '{sep_type}', expected 'Voluntary Resignation'\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"David Kim (E-1002): {'; '.join(issues)}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"David Kim (E-1002) row verified: Separated, MM/DD/YYYY 04/06/2026, Voluntary Resignation.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "57ef7283-93c8-47cd-a493-6457e21230bc",
|
||
"sort_order": 10,
|
||
"rubric_text": "In `slack_data.json`, the `#hr-offboarding` channel must contain a message with ALL of: 'OFFBOARDING COMPLETE', 'David Kim', 'E-1002', 'Voluntary Resignation', 'Last Day:', '04/06/2026'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping check.\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"slack_data.json not found; skipping check.\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n target_channel_id = None\n for cid, cinfo in channels.items():\n if cinfo.get(\"name\", \"\").lower() == \"hr-offboarding\":\n target_channel_id = cid\n break\n if target_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-offboarding not found in slack_data.json\"}\n messages = data.get(\"messages\", {}).get(target_channel_id, [])\n required_terms = [\"OFFBOARDING COMPLETE\", \"David Kim\", \"E-1002\", \"Voluntary Resignation\", \"Last Day:\", \"04/06/2026\"]\n for msg in messages:\n text = msg.get(\"text\", \"\")\n if all(term in text for term in required_terms):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found message in #hr-offboarding with all required David Kim offboarding terms.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in #hr-offboarding contains all of: {required_terms}\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "4bccaed5-3de3-4419-9390-bc2d4544f222",
|
||
"sort_order": 11,
|
||
"rubric_text": "In `mailbox.json`, email with id=2 must have `is_read == true` and `folder == 'Offboarding'`.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping check.\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; skipping check.\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n email = next((e for e in data.get(\"emails\", []) if str(e.get(\"email_id\", \"\")) == \"2\"), None)\n if email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email with id=2 not found.\"}\n issues = []\n if not email.get(\"is_read\", False):\n issues.append(\"is_read is not True\")\n folder = email.get(\"folder\", \"\")\n if folder.lower() != \"offboarding\":\n issues.append(f\"folder is '{folder}', expected 'Offboarding'\")\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email id=2: {'; '.join(issues)}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email id=2 is_read=True and folder='Offboarding'.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776042091132",
|
||
"sort_order": 12,
|
||
"rubric_text": "No Jira issue exists with a summary containing \"Okafor\".",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; skipping check.\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found at expected path.\"}\n with open(jira_path) as f:\n data = json.load(f)\n issues = data.get(\"issues\", {})\n all_issues = list(issues.values()) if isinstance(issues, dict) else issues\n\n def get_summary(i):\n return (i.get(\"fields\", {}).get(\"summary\") or i.get(\"summary\") or \"\").strip()\n\n okafor_issues = [get_summary(i) for i in all_issues if \"okafor\" in get_summary(i).lower()]\n if okafor_issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found Jira issue(s) with summary containing 'Okafor': {okafor_issues}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No Jira issue found with a summary containing 'Okafor'.\"}",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776042100472",
|
||
"sort_order": 13,
|
||
"rubric_text": "No sent email exists with `to_addr == j.okafor@crestwood.edu`. No sent email exists with a subject or body mentioning \"Okafor\" or \"E-1003\".",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; cannot check sent emails, so passing by default.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; no sent emails to check, so passing.\"}\n\n try:\n with open(mailbox_path, 'r') as f:\n mailbox = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading mailbox.json: {e}\"}\n\n emails = mailbox.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n if not sent_emails:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails found at all. Rubric satisfied.\"}\n\n issues = []\n\n for email in sent_emails:\n to_addr = (email.get(\"to_addr\", \"\") or \"\").strip().lower()\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n email_id = email.get(\"email_id\", \"unknown\")\n\n # Check 1: No sent email to j.okafor@crestwood.edu\n if to_addr == \"j.okafor@crestwood.edu\":\n issues.append(f\"Sent email (id={email_id}) has to_addr='j.okafor@crestwood.edu'.\")\n\n # Check 2: No sent email mentioning \"Okafor\" or \"E-1003\" in subject or body\n for keyword in [\"okafor\", \"e-1003\"]:\n if keyword in subject:\n issues.append(f\"Sent email (id={email_id}) subject contains '{keyword}'.\")\n if keyword in body:\n issues.append(f\"Sent email (id={email_id}) body contains '{keyword}'.\")\n\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Rubric FAILED. Found sent emails violating the conditions:\\n\" + \"\\n\".join(issues)\n }\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Rubric satisfied. Checked {len(sent_emails)} sent email(s); none were addressed to j.okafor@crestwood.edu and none mention 'Okafor' or 'E-1003'.\"\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776042123728",
|
||
"sort_order": 14,
|
||
"rubric_text": "No Slack message in any channel mentions \"Okafor\" or \"E-1003\".",
|
||
"verifier_code": "from pathlib import Path \nimport json \nimport re \n\ndef verify(workspace_path, external_services_path=None): \n slack_path = None \n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\" \n if candidate.exists():\n slack_path = candidate \n \n if slack_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"slack_data.json not found in external services. Passing by default (no evidence of violation).\"}\n \n # Search for JSON files that might contain Slack messages \n found_mentions = [] \n files_checked = 0 \n \n # Look for all JSON files under the slack path \n json_files = list(slack_path.rglob(\"*.json\")) if slack_path.is_dir() else [slack_path] \n \n for json_file in json_files: \n try:\n with open(json_file, 'r', encoding='utf-8') as f: \n data = json.load(f) \n files_checked += 1\n \n # Recursively search for message content\n def search_data(obj, file_path, path=\"\"): \n if isinstance(obj, dict): \n # Check if this dict has a message-like structure\n text_fields = [] \n for key in ['text', 'message', 'body', 'content', 'msg']: \n if key in obj and isinstance(obj[key], str): \n text_fields.append((key, obj[key])) \n \n for key, text in text_fields: \n text_lower = text.lower() \n if 'okafor' in text_lower or 'e-1003' in text_lower: \n channel = obj.get('channel', obj.get('channel_name', path)) \n found_mentions.append({ \n 'file': str(file_path), \n 'channel': channel, \n 'text_snippet': text[:200], \n 'matched': [] \n })\n if 'okafor' in text_lower: \n found_mentions[-1]['matched'].append('Okafor') \n if 'e-1003' in text_lower:\n found_mentions[-1]['matched'].append('E-1003') \n \n for k, v in obj.items(): \n search_data(v, file_path, f\"{path}/{k}\") \n elif isinstance(obj, list): \n for i, item in enumerate(obj):\n search_data(item, file_path, f\"{path}[{i}]\") \n elif isinstance(obj, str): \n obj_lower = obj.lower()\n if 'okafor' in obj_lower or 'e-1003' in obj_lower: \n # Only flag if it looks like a message context \n if len(obj) > 5: # avoid matching random short strings\n found_mentions.append({ \n 'file': str(file_path), \n 'channel': path, \n 'text_snippet': obj[:200], \n 'matched': [] \n }) \n if 'okafor' in obj_lower:\n found_mentions[-1]['matched'].append('Okafor') \n if 'e-1003' in obj_lower:\n found_mentions[-1]['matched'].append('E-1003') \n \n search_data(data, json_file)\n except (json.JSONDecodeError, UnicodeDecodeError, IOError): \n continue \n\n if found_mentions: \n details = [] \n for m in found_mentions[:5]: # Show up to 5 examples\n details.append(f\" - Matched {m['matched']} in {m['file']}: \\\"{m['text_snippet']}...\\\"\") \n detail_str = \"\\n\".join(details) \n return { \n \"pass\": False, \n \"score\": 0.0, \n \"feedback\": f\"FAIL: Found {len(found_mentions)} Slack message(s) mentioning 'Okafor' or 'E-1003':\\n{detail_str}\" \n } \n else:\n return { \n \"pass\": True, \n \"score\": 1.0, \n \"feedback\": f\"PASS: Checked {files_checked} Slack data file(s) under '{slack_path}'. No messages mention 'Okafor' or 'E-1003'.\" \n } ",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776042175636",
|
||
"sort_order": 15,
|
||
"rubric_text": "- `employee_roster.xlsx` Sheet1: The row for James Okafor (E-1003) has `status == \"Active\"`.\n- `benefits_enrollment.xlsx` Sheet1: The row for James Okafor (E-1003) has `status == \"Active\"`.\n- `benefits_enrollment.xlsx` Sheet1: The row for James Okafor (E-1003) has `cobra_notification_date` empty/None.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n workspace_path = Path(workspace_path)\n feedback_parts = []\n all_pass = True\n\n # Helper: find column index by header name (case-insensitive, stripped)\n def find_col(headers, name):\n name_lower = name.strip().lower()\n for i, h in enumerate(headers):\n if h is not None and str(h).strip().lower() == name_lower:\n return i\n return None\n\n def cell_matches(val, target):\n if val is None:\n return False\n return str(val).strip().lower() == str(target).strip().lower()\n\n def is_empty(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == '' or s.lower() in ('none', 'n/a', 'na', 'null', 'nat')\n\n # --- Check 1: employee_roster.xlsx ---\n roster_file = workspace_path / 'employee_roster.xlsx'\n if not roster_file.exists():\n all_pass = False\n feedback_parts.append(f\"FAIL: '{roster_file.name}' not found in workspace.\")\n else:\n wb = openpyxl.load_workbook(roster_file, data_only=True)\n ws = wb[wb.sheetnames[0]] # Sheet1\n rows = list(ws.iter_rows(values_only=True))\n if len(rows) < 2:\n all_pass = False\n feedback_parts.append(f\"FAIL: '{roster_file.name}' has no data rows.\")\n else:\n headers = [str(h).strip().lower() if h is not None else '' for h in rows[0]]\n # find relevant columns\n emp_id_col = None\n name_col = None\n status_col = None\n for i, h in enumerate(headers):\n if 'employee' in h and 'id' in h:\n emp_id_col = i\n elif h in ('name', 'employee_name', 'employee name', 'full_name', 'full name'):\n name_col = i\n elif h == 'first_name' or h == 'first name':\n name_col = i # fallback\n if h == 'status':\n status_col = i\n\n found_row = None\n for row in rows[1:]:\n match = False\n if emp_id_col is not None and row[emp_id_col] is not None:\n if str(row[emp_id_col]).strip().upper().replace(' ', '') in ('E-1003', 'E1003'):\n match = True\n if not match and name_col is not None and row[name_col] is not None:\n if 'okafor' in str(row[name_col]).strip().lower() and 'james' in str(row[name_col]).strip().lower():\n match = True\n if not match:\n # brute force: check all cells for E-1003 or James Okafor\n row_str = ' '.join([str(c).strip().lower() for c in row if c is not None])\n if ('e-1003' in row_str or 'e1003' in row_str) or ('james' in row_str and 'okafor' in row_str):\n match = True\n if match:\n found_row = row\n break\n\n if found_row is None:\n all_pass = False\n feedback_parts.append(f\"FAIL: No row for James Okafor (E-1003) found in '{roster_file.name}'.\")\n else:\n if status_col is not None:\n status_val = found_row[status_col]\n if status_val is not None and str(status_val).strip().lower() == 'active':\n feedback_parts.append(f\"PASS: '{roster_file.name}' - James Okafor status is 'Active' (found: '{status_val}').\")\n else:\n all_pass = False\n feedback_parts.append(f\"FAIL: '{roster_file.name}' - James Okafor status expected 'Active', found '{status_val}'.\")\n else:\n # Try to find status in row by scanning headers more broadly\n all_pass = False\n feedback_parts.append(f\"FAIL: '{roster_file.name}' - Could not find 'status' column. Headers: {headers}\")\n wb.close()\n\n # --- Check 2 & 3: benefits_enrollment.xlsx ---\n benefits_file = workspace_path / 'benefits_enrollment.xlsx'\n if not benefits_file.exists():\n all_pass = False\n feedback_parts.append(f\"FAIL: '{benefits_file.name}' not found in workspace.\")\n else:\n wb = openpyxl.load_workbook(benefits_file, data_only=True)\n ws = wb[wb.sheetnames[0]] # Sheet1\n rows = list(ws.iter_rows(values_only=True))\n if len(rows) < 2:\n all_pass = False\n feedback_parts.append(f\"FAIL: '{benefits_file.name}' has no data rows.\")\n else:\n headers = [str(h).strip().lower() if h is not None else '' for h in rows[0]]\n emp_id_col = None\n name_col = None\n status_col = None\n cobra_col = None\n for i, h in enumerate(headers):\n if 'employee' in h and 'id' in h:\n emp_id_col = i\n if h in ('name', 'employee_name', 'employee name', 'full_name', 'full name'):\n name_col = i\n if h == 'first_name' or h == 'first name':\n if name_col is None:\n name_col = i\n if h == 'status':\n status_col = i\n if 'cobra' in h and 'notif' in h:\n cobra_col = i\n\n found_row = None\n for row in rows[1:]:\n match = False\n if emp_id_col is not None and row[emp_id_col] is not None:\n if str(row[emp_id_col]).strip().upper().replace(' ', '') in ('E-1003', 'E1003'):\n match = True\n if not match and name_col is not None and row[name_col] is not None:\n if 'okafor' in str(row[name_col]).strip().lower() and 'james' in str(row[name_col]).strip().lower():\n match = True\n if not match:\n row_str = ' '.join([str(c).strip().lower() for c in row if c is not None])\n if ('e-1003' in row_str or 'e1003' in row_str) or ('james' in row_str and 'okafor' in row_str):\n match = True\n if match:\n found_row = row\n break\n\n if found_row is None:\n all_pass = False\n feedback_parts.append(f\"FAIL: No row for James Okafor (E-1003) found in '{benefits_file.name}'.\")\n else:\n # Check status == Active\n if status_col is not None:\n status_val = found_row[status_col]\n if status_val is not None and str(status_val).strip().lower() == 'active':\n feedback_parts.append(f\"PASS: '{benefits_file.name}' - James Okafor status is 'Active' (found: '{status_val}').\")\n else:\n all_pass = False\n feedback_parts.append(f\"FAIL: '{benefits_file.name}' - James Okafor status expected 'Active', found '{status_val}'.\")\n else:\n all_pass = False\n feedback_parts.append(f\"FAIL: '{benefits_file.name}' - Could not find 'status' column. Headers: {headers}\")\n\n # Check cobra_notification_date is empty/None\n if cobra_col is not None:\n cobra_val = found_row[cobra_col]\n if is_empty(cobra_val):\n feedback_parts.append(f\"PASS: '{benefits_file.name}' - James Okafor cobra_notification_date is empty (found: '{cobra_val}').\")\n else:\n all_pass = False\n feedback_parts.append(f\"FAIL: '{benefits_file.name}' - James Okafor cobra_notification_date expected empty/None, found '{cobra_val}'.\")\n else:\n # Maybe the column doesn't exist yet, which means it's effectively empty\n feedback_parts.append(f\"PASS (lenient): '{benefits_file.name}' - No 'cobra_notification_date' column found, treating as empty for James Okafor.\")\n wb.close()\n\n score = 1.0 if all_pass else 0.0\n feedback = '\\n'.join(feedback_parts)\n return {\"pass\": all_pass, \"score\": score, \"feedback\": feedback}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776042204299",
|
||
"sort_order": 16,
|
||
"rubric_text": "Email with id=3 remains in the \"INBOX\" folder.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n mailbox = json.load(f)\n \n emails = mailbox.get(\"emails\", [])\n \n target_email = None\n for email in emails:\n eid = email.get(\"email_id\", \"\")\n if str(eid).strip() == \"3\":\n target_email = email\n break\n \n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email with id=3 was not found in the mailbox at all.\"}\n \n folder = target_email.get(\"folder\", \"\").strip()\n if folder.upper() == \"INBOX\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email id=3 is in folder '{folder}' as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email id=3 is in folder '{folder}', expected 'INBOX'.\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776523213132",
|
||
"sort_order": 17,
|
||
"rubric_text": "In `jira_state.json`, the top-level `comments` dict must contain at least one comment associated with `OFFBOARD-1` whose text contains both `11.4` and `Access` (case-insensitive).",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external services path provided; cannot check jira_state.json.\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found at expected path.\"}\n with open(jira_path, encoding=\"utf-8\") as f:\n data = json.load(f)\n\n top_comments = data.get(\"comments\", {})\n if not isinstance(top_comments, dict):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 'comments' to be a dict, got {type(top_comments).__name__}.\"}\n\n offboard1_comments = top_comments.get(\"OFFBOARD-1\", [])\n if not offboard1_comments:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No comments found under 'OFFBOARD-1' in the top-level comments dict.\"}\n\n def extract_text(comment):\n body = comment.get(\"body\", {})\n if isinstance(body, dict):\n try:\n return body[\"content\"][0][\"content\"][0][\"text\"]\n except (KeyError, IndexError, TypeError):\n return str(body)\n return str(body)\n\n for comment in offboard1_comments:\n text = extract_text(comment)\n if \"[11.4]\" in text and \"access\" in text.lower():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found comment on OFFBOARD-1 containing both '[11.4]' and 'Access': '{text[:200]}'\"}\n\n texts = [extract_text(c)[:100] for c in offboard1_comments]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(offboard1_comments)} comment(s) on OFFBOARD-1 but none contain both '[11.4]' and 'Access'. Comments: {texts}\"}",
|
||
"criterion_type": "expected_output"
|
||
}
|
||
]
|