Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,149 @@
|
||||
[
|
||||
{
|
||||
"id": "71217907-7deb-40e9-9093-420a3711b96c",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `employee_roster.xlsx` Sheet1, rows for Daniel Yoon (department='Marketing', campus='West'), Priya Kapoor (department='Finance', campus='Main'), and Marcus Webb (department='Information Technology', campus='Main') must exist with non-empty employee_id values.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n roster_files = [f for f in xlsx_files if 'employee_roster' in f.name.lower()]\n if not roster_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No employee_roster.xlsx file found in workspace'}\n \n wb = openpyxl.load_workbook(roster_files[0])\n ws = wb.worksheets[0]\n \n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n \n def find_col(names):\n for name in names:\n for i, h in enumerate(headers):\n if name in h:\n return i\n return None\n \n first_name_col = find_col(['first_name', 'first name', 'firstname'])\n last_name_col = find_col(['last_name', 'last name', 'lastname'])\n name_col = find_col(['name']) if first_name_col is None else None\n emp_id_col = find_col(['employee_id', 'employee id', 'emp_id', 'empid'])\n dept_col = find_col(['department', 'dept'])\n campus_col = find_col(['campus'])\n \n required = [\n {'first': 'Daniel', 'last': 'Yoon', 'dept': 'marketing', 'campus': 'west'},\n {'first': 'Priya', 'last': 'Kapoor', 'dept': 'finance', 'campus': 'main'},\n {'first': 'Marcus', 'last': 'Webb', 'dept': 'information technology', 'campus': 'main'},\n ]\n \n found = {r['last']: False for r in required}\n valid_dept = {r['last']: False for r in required}\n valid_campus = {r['last']: False for r in required}\n valid_id = {r['last']: False for r in required}\n \n for row in ws.iter_rows(min_row=2, values_only=True):\n row = list(row)\n if first_name_col is not None and last_name_col is not None:\n fname = str(row[first_name_col]).strip() if row[first_name_col] else ''\n lname = str(row[last_name_col]).strip() if row[last_name_col] else ''\n elif name_col is not None:\n parts = str(row[name_col]).strip().split() if row[name_col] else []\n fname = parts[0] if parts else ''\n lname = parts[-1] if len(parts) > 1 else ''\n else:\n continue\n \n for req in required:\n if fname.lower() == req['first'].lower() and lname.lower() == req['last'].lower():\n found[req['last']] = True\n emp_id = str(row[emp_id_col]).strip() if emp_id_col is not None and row[emp_id_col] else ''\n dept = str(row[dept_col]).strip().lower() if dept_col is not None and row[dept_col] else ''\n campus = str(row[campus_col]).strip().lower() if campus_col is not None and row[campus_col] else ''\n if emp_id and emp_id.lower() not in ('none', 'null', ''):\n valid_id[req['last']] = True\n if req['dept'] in dept or dept in req['dept']:\n valid_dept[req['last']] = True\n if req['campus'] in campus or campus in req['campus']:\n valid_campus[req['last']] = True\n \n issues = []\n for req in required:\n last = req['last']\n if not found[last]:\n issues.append(f'{req[\"first\"]} {last} not found in roster')\n else:\n if not valid_id[last]:\n issues.append(f'{last} has empty employee_id')\n if not valid_dept[last]:\n issues.append(f'{last} department mismatch (expected {req[\"dept\"]})')\n if not valid_campus[last]:\n issues.append(f'{last} campus mismatch (expected {req[\"campus\"]})')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': 'Issues: ' + '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'All three employees (Yoon/Marketing/West, Kapoor/Finance/Main, Webb/IT/Main) found with valid employee_ids'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "7fa37e37-4f95-4040-970b-d7e8e4f39065",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `onboarding_checklist.xlsx` Sheet1, rows must exist for Daniel Yoon, Priya Kapoor, and Marcus Webb.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n checklist_files = [f for f in xlsx_files if 'onboarding_checklist' in f.name.lower()]\n if not checklist_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No onboarding_checklist.xlsx file found in workspace'}\n \n wb = openpyxl.load_workbook(checklist_files[0])\n ws = wb.worksheets[0]\n \n required = [('Daniel', 'Yoon'), ('Priya', 'Kapoor'), ('Marcus', 'Webb')]\n found = {last: False for _, last in required}\n \n for row in ws.iter_rows(min_row=1, values_only=True):\n row_text = ' '.join(str(c) for c in row if c is not None).lower()\n for first, last in required:\n if first.lower() in row_text and last.lower() in row_text:\n found[last] = True\n \n missing = [last for last in found if not found[last]]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing rows in onboarding_checklist.xlsx for: {missing}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'Rows found for Daniel Yoon, Priya Kapoor, and Marcus Webb in onboarding_checklist.xlsx'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "ac0968ac-d1b5-49a1-981f-1bbfff91b6db",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `jira_state.json`, exactly one issue must exist with summary 'ONBOARD-Yoon-04282026', exactly one with 'ONBOARD-Kapoor-05052026', and exactly one with 'ONBOARD-Webb-04282026'.",
|
||||
"verifier_code": "from pathlib import Path \nimport json \n \ndef verify(workspace_path, external_services_path=None): \n if not external_services_path: \n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n \n jira_path = Path(external_services_path) / 'jira_state.json'\n if not jira_path.exists(): \n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found'}\n \n with open(jira_path) as f:\n data = json.load(f) \n \n issues = data.get('issues', {})\n if isinstance(issues, dict):\n all_issues = list(issues.values())\n elif isinstance(issues, list): \n all_issues = issues\n else: \n all_issues = [] \n\n required = ['ONBOARD-Yoon-04282026', 'ONBOARD-Kapoor-05052026', 'ONBOARD-Webb-04282026'] \n counts = {r: 0 for r in required}\n \n for issue in all_issues: \n summary = str(issue.get('fields', {}).get('summary', '')).strip()\n for req in required:\n if summary == req: \n counts[req] += 1\n \n issues_list = [] \n for req in required:\n if counts[req] == 0:\n issues_list.append(f'No Jira issue with summary \"{req}\"')\n elif counts[req] > 1: \n issues_list.append(f'Multiple Jira issues ({counts[req]}) with summary \"{req}\"')\n \n if issues_list: \n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues_list)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Found exactly one each of ONBOARD-Yoon-04282026, ONBOARD-Kapoor-05052026, ONBOARD-Webb-04282026'}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "914baf74-23f4-4899-95c6-d19bd194e79d",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `slack_data.json`, channel `#hr-onboarding` must contain messages containing ALL of: ('NEW HIRE','Daniel Yoon','Marketing Coordinator','Marketing','West','04/28/2026'), ('NEW HIRE','Priya Kapoor','Finance Analyst','Finance','Main','05/05/2026'), and ('NEW HIRE','Marcus Webb','IT Support Specialist','Information Technology','Main','04/28/2026').",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n \n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n channels = data.get('channels', {})\n channel_id = None\n for cid, cinfo in channels.items():\n if cinfo.get('name', '').lower() == 'hr-onboarding':\n channel_id = cid\n break\n \n if not channel_id:\n return {'pass': False, 'score': 0.0, 'feedback': '#hr-onboarding channel not found in Slack'}\n \n messages = data.get('messages', {}).get(channel_id, [])\n all_text = ' '.join(str(m.get('text', '')) for m in messages)\n \n required_sets = [\n ['NEW HIRE', 'Daniel Yoon', 'Marketing Coordinator', 'Marketing', 'West', '04/28/2026'],\n ['NEW HIRE', 'Priya Kapoor', 'Finance Analyst', 'Finance', 'Main', '05/05/2026'],\n ['NEW HIRE', 'Marcus Webb', 'IT Support Specialist', 'Information Technology', 'Main', '04/28/2026'],\n ]\n \n missing = []\n for req_set in required_sets:\n # Check each message for all terms in a single message\n found = False\n for msg in messages:\n text = str(msg.get('text', ''))\n if all(term.lower() in text.lower() for term in req_set):\n found = True\n break\n if not found:\n missing.append(str(req_set))\n \n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing #hr-onboarding messages containing all required terms: {missing}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'All three new hire announcements found in #hr-onboarding'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "49d7907f-e047-4120-aded-6170ce85c32e",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `mailbox.json`, sent emails must exist: to danielyoon.work@gmail.com with subject containing 'New Hire Welcome' and 'Yoon'; to priya.kapoor@outlook.com with subject containing 'New Hire Welcome' and 'Kapoor'; to marcuswebb84@gmail.com with subject containing 'New Hire Welcome' and 'Webb'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n \n required = [\n {'to': 'danielyoon.work@gmail.com', 'subj_terms': ['new hire welcome', 'yoon']},\n {'to': 'priya.kapoor@outlook.com', 'subj_terms': ['new hire welcome', 'kapoor']},\n {'to': 'marcuswebb84@gmail.com', 'subj_terms': ['new hire welcome', 'webb']},\n ]\n \n missing = []\n for req in required:\n found = False\n for email in sent_emails:\n to_addr = str(email.get('to_addr', '')).lower()\n subj = str(email.get('subject', '')).lower()\n if req['to'].lower() in to_addr and all(t in subj for t in req['subj_terms']):\n found = True\n break\n if not found:\n missing.append(f\"to {req['to']} with subject containing {req['subj_terms']}\")\n \n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing sent welcome emails: {missing}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'Welcome emails found for Yoon, Kapoor, and Webb'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4230052e-275b-4cb2-9df4-92903ca05f86",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `mailbox.json`, sent emails to j.park@crestwood.edu must exist with subject containing 'IT Access Provisioning Request' and each of 'Yoon', 'Kapoor', 'Webb' (one email each).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n \n required_names = ['yoon', 'kapoor', 'webb']\n missing = []\n for name in required_names:\n found = False\n for email in sent_emails:\n to_addr = str(email.get('to_addr', '')).lower()\n subj = str(email.get('subject', '')).lower()\n body = str(email.get('body_text', '')).lower()\n if 'j.park@crestwood.edu' in to_addr and 'it access provisioning request' in subj and name in (subj + body):\n found = True\n break\n if not found:\n missing.append(f'IT Access Provisioning Request to j.park@crestwood.edu mentioning {name}')\n \n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing emails: {missing}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'IT Access Provisioning Request emails sent to j.park@crestwood.edu for Yoon, Kapoor, Webb'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "a7a434a0-2c60-43e5-a7de-7f9e95dd0a11",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `mailbox.json`, orientation coordinator request emails must exist: to s.bloom@crestwood.edu with subject containing 'Orientation Coordinator Request' and 'Yoon'; to r.chen@crestwood.edu with subject containing 'Orientation Coordinator Request' and 'Kapoor'; to l.nguyen@crestwood.edu with subject containing 'Orientation Coordinator Request' and 'Webb'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n \n required = [\n {'to': 's.bloom@crestwood.edu', 'name': 'yoon'},\n {'to': 'r.chen@crestwood.edu', 'name': 'kapoor'},\n {'to': 'l.nguyen@crestwood.edu', 'name': 'webb'},\n ]\n \n missing = []\n for req in required:\n found = False\n for email in sent_emails:\n to_addr = str(email.get('to_addr', '')).lower()\n subj = str(email.get('subject', '')).lower()\n body = str(email.get('body_text', '')).lower()\n if req['to'].lower() in to_addr and 'orientation coordinator request' in subj and req['name'] in (subj + body):\n found = True\n break\n if not found:\n missing.append(f\"Orientation Coordinator Request to {req['to']} mentioning {req['name']}\")\n \n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing emails: {missing}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'Orientation Coordinator Request emails sent to s.bloom (Yoon), r.chen (Kapoor), l.nguyen (Webb)'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "f25ad0db-395e-45c5-bd93-8711356254fe",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `calendar_data.json`, orientation events must exist: one containing 'Orientation' and 'Daniel Yoon' starting 2026-04-28 at 9:00am ending 12:00pm; one containing 'Orientation' and 'Priya Kapoor' starting 2026-05-05 at 9:00am ending 12:00pm; one containing 'Orientation' and 'Marcus Webb' starting 2026-04-28 at 9:00am ending 12:00pm.",
|
||||
"verifier_code": "from pathlib import Path \nimport json \n\ndef verify(workspace_path, external_services_path=None): \n if not external_services_path: \n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n \n cal_path = Path(external_services_path) / 'calendar_data.json'\n if not cal_path.exists(): \n return {'pass': False, 'score': 0.0, 'feedback': 'calendar_data.json not found'}\n \n with open(cal_path) as f:\n data = json.load(f) \n \n events = data.get('events', {})\n if isinstance(events, dict):\n all_events = list(events.values()) \n elif isinstance(events, list):\n all_events = events \n else: \n all_events = []\n\n required = [ \n {'name': 'Daniel Yoon', 'date': '2026-04-28'},\n {'name': 'Priya Kapoor', 'date': '2026-05-05'}, \n {'name': 'Marcus Webb', 'date': '2026-04-28'}, \n ]\n \n missing = [] \n for req in required:\n found = False\n for event in all_events:\n summary = str(event.get('summary', '')).lower() \n start = event.get('start', {})\n end = event.get('end', {}) \n start_str = start.get('dateTime', '') if isinstance(start, dict) else str(start)\n end_str = end.get('dateTime', '') if isinstance(end, dict) else str(end) \n \n if 'orientation' not in summary or req['name'].lower() not in summary: \n continue \n if req['date'] not in start_str: \n continue\n if 'T09:00' not in start_str and 'T9:00' not in start_str: \n continue \n if 'T12:00' not in end_str:\n continue \n found = True \n break \n if not found: \n missing.append(f\"Orientation event for {req['name']} on {req['date']} 9am-12pm\")\n \n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing calendar events: {missing}'} \n return {'pass': True, 'score': 1.0, 'feedback': 'Orientation calendar events found for Daniel Yoon (4/28), Priya Kapoor (5/5), Marcus Webb (4/28)'} ",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e2826577-7298-4d4b-886e-cfe2ea8cb7d5",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `employee_roster.xlsx` Sheet1, exactly one row with last_name 'Ellis' or employee_id 'E-1012' must exist, and no row for 'Sarah Mendez' must exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n roster_files = [f for f in xlsx_files if 'employee_roster' in f.name.lower()]\n if not roster_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No employee_roster.xlsx found'}\n \n wb = openpyxl.load_workbook(roster_files[0])\n ws = wb.worksheets[0]\n \n ellis_count = 0\n mendez_found = False\n \n for row in ws.iter_rows(min_row=2, values_only=True):\n row_text = ' '.join(str(c) for c in row if c is not None).lower()\n if 'ellis' in row_text or 'e-1012' in row_text:\n ellis_count += 1\n if 'mendez' in row_text:\n mendez_found = True\n \n issues = []\n if ellis_count == 0:\n issues.append('No row for Ellis (or employee_id E-1012) found in employee_roster.xlsx')\n elif ellis_count > 1:\n issues.append(f'Multiple rows ({ellis_count}) for Ellis found in employee_roster.xlsx (expected exactly 1)')\n if mendez_found:\n issues.append('Row for Sarah Mendez found in employee_roster.xlsx (should not exist)')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Exactly one Ellis row found and no Mendez row in employee_roster.xlsx'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "8df32160-aab8-46be-af83-75a2017bd497",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `onboarding_checklist.xlsx` Sheet1, exactly one row for Brandon Ellis must exist and no row for Sarah Mendez must exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n checklist_files = [f for f in xlsx_files if 'onboarding_checklist' in f.name.lower()]\n if not checklist_files:\n return {'pass': False, 'score': 0.0, 'feedback': 'No onboarding_checklist.xlsx found'}\n \n wb = openpyxl.load_workbook(checklist_files[0])\n ws = wb.worksheets[0]\n \n ellis_count = 0\n mendez_found = False\n \n for row in ws.iter_rows(min_row=1, values_only=True):\n row_text = ' '.join(str(c) for c in row if c is not None).lower()\n if 'brandon' in row_text and 'ellis' in row_text:\n ellis_count += 1\n if 'mendez' in row_text:\n mendez_found = True\n \n issues = []\n if ellis_count == 0:\n issues.append('No row for Brandon Ellis found in onboarding_checklist.xlsx')\n elif ellis_count > 1:\n issues.append(f'Multiple rows ({ellis_count}) for Brandon Ellis in onboarding_checklist.xlsx (expected 1)')\n if mendez_found:\n issues.append('Row for Sarah Mendez found in onboarding_checklist.xlsx (should not exist)')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Exactly one Brandon Ellis row and no Mendez row in onboarding_checklist.xlsx'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "d867b689-64ea-4683-bffc-a1e8ae7e104b",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In `jira_state.json`, exactly one issue with summary containing 'ONBOARD-Ellis' must exist, and no issue with summary containing 'ONBOARD-Mendez' must exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n jira_path = Path(external_services_path) / 'jira_state.json'\n if not jira_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found'}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n issues = data.get('issues', {})\n if isinstance(issues, dict):\n all_issues = list(issues.values())\n elif isinstance(issues, list):\n all_issues = issues\n else:\n all_issues = []\n\n ellis_count = sum(1 for i in all_issues if 'onboard-ellis' in str(i.get('fields', {}).get('summary', '')).lower())\n mendez_count = sum(1 for i in all_issues if 'onboard-mendez' in str(i.get('fields', {}).get('summary', '')).lower())\n\n issues_list = []\n if ellis_count == 0:\n issues_list.append('No Jira issue with summary containing ONBOARD-Ellis found')\n elif ellis_count > 1:\n issues_list.append(f'Multiple ({ellis_count}) Jira issues with ONBOARD-Ellis (expected exactly 1)')\n if mendez_count > 0:\n issues_list.append(f'Jira issue with ONBOARD-Mendez found (should not exist)')\n\n if issues_list:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues_list)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Exactly one ONBOARD-Ellis Jira issue and no ONBOARD-Mendez issue found'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e96a155d-f34d-4af9-a0ab-539d4848f0ff",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "In `slack_data.json`, channel `#hr-onboarding` must contain exactly one message mentioning 'Brandon Ellis'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n\n with open(slack_path) as f:\n data = json.load(f)\n\n channels = data.get('channels', {})\n channel_id = None\n for cid, cinfo in channels.items():\n if cinfo.get('name', '').lower() == 'hr-onboarding':\n channel_id = cid\n break\n\n if not channel_id:\n return {'pass': False, 'score': 0.0, 'feedback': '#hr-onboarding channel not found'}\n\n messages = data.get('messages', {}).get(channel_id, [])\n ellis_msgs = [m for m in messages if 'brandon ellis' in str(m.get('text', '')).lower()]\n\n if len(ellis_msgs) == 0:\n return {'pass': False, 'score': 0.0, 'feedback': 'No message mentioning Brandon Ellis in #hr-onboarding'}\n if len(ellis_msgs) > 1:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found {len(ellis_msgs)} messages mentioning Brandon Ellis in #hr-onboarding (expected exactly 1)'}\n return {'pass': True, 'score': 1.0, 'feedback': 'Exactly one message mentioning Brandon Ellis in #hr-onboarding'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "5831154f-6588-42d0-be73-ad43a167c4f9",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In `mailbox.json`, exactly one sent email to brandon.ellis.personal@gmail.com must exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n issues = []\n\n if mailbox_path.exists():\n with open(mailbox_path) as f:\n mail_data = json.load(f)\n sent_emails = [e for e in mail_data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n ellis_emails = [e for e in sent_emails if 'brandon.ellis.personal@gmail.com' in str(e.get('to_addr','')).lower()]\n if len(ellis_emails) == 0:\n issues.append('No sent email to brandon.ellis.personal@gmail.com found')\n elif len(ellis_emails) > 1:\n issues.append(f'Found {len(ellis_emails)} sent emails to brandon.ellis.personal@gmail.com (expected exactly 1)')\n else:\n issues.append('mailbox.json not found')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Exactly one sent email to brandon.ellis.personal@gmail.com'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "6cbd2d97-2f99-48c8-88b0-77391ccfb370",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `calendar_data.json`, exactly one event mentioning 'Ellis' must exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n cal_path = Path(external_services_path) / 'calendar_data.json'\n issues = []\n\n if cal_path.exists():\n with open(cal_path) as f:\n cal_data = json.load(f)\n events = cal_data.get('events', {})\n if isinstance(events, dict):\n all_events = list(events.values())\n elif isinstance(events, list):\n all_events = events\n else:\n all_events = []\n ellis_events = [e for e in all_events if 'ellis' in str(e.get('summary', '')).lower()]\n if len(ellis_events) == 0:\n issues.append('No calendar event mentioning Ellis found')\n elif len(ellis_events) > 1:\n issues.append(f'Found {len(ellis_events)} calendar events mentioning Ellis (expected exactly 1)')\n else:\n issues.append('calendar_data.json not found')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Exactly one calendar event mentioning Ellis'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "3651aa78-2855-4d62-9c6b-f2dd9e1778d7",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "In `calendar_data.json`, no events after 04-10-26 mentioning 'Mendez' must exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check calendar_data.json'}\n \n cal_path = Path(external_services_path) / 'calendar_data.json'\n if not cal_path.exists():\n # If no calendar data file exists, there are no Mendez events\n return {'pass': True, 'score': 1.0, 'feedback': 'calendar_data.json not found; no Mendez events exist by default'}\n \n with open(cal_path) as f:\n cal_data = json.load(f)\n \n events = cal_data.get('events', {})\n if isinstance(events, dict):\n all_events = list(events.values())\n elif isinstance(events, list):\n all_events = events\n else:\n all_events = []\n \n # The cutoff date: 04-10-26 interpreted as 2026-04-10\n cutoff_date = datetime(2026, 4, 10)\n \n # Find events that mention 'mendez' (case-insensitive) AND are after the cutoff date\n mendez_events_after_cutoff = []\n mendez_events_on_or_before = []\n \n for e in all_events:\n event_str = json.dumps(e).lower()\n if 'mendez' not in event_str:\n continue\n \n # Try to extract date from event fields\n event_date = None\n for date_field in ['start', 'date', 'start_date', 'startDate', 'dateTime', 'start_time']:\n val = e.get(date_field, '')\n if isinstance(val, dict):\n val = val.get('dateTime', val.get('date', ''))\n if isinstance(val, str) and val:\n # Try multiple date formats\n for fmt in ['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%dT%H:%M:%S%z', '%Y-%m-%dT%H:%M', '%m/%d/%Y', '%m-%d-%Y', '%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%dT%H:%M:%S.%f']:\n try:\n parsed = datetime.strptime(val.split('+')[0].split('Z')[0].strip(), fmt.replace('%z', ''))\n event_date = parsed\n break\n except ValueError:\n continue\n # Also try regex to extract date\n if event_date is None:\n match = re.search(r'(\\d{4})-(\\d{2})-(\\d{2})', val)\n if match:\n try:\n event_date = datetime(int(match.group(1)), int(match.group(2)), int(match.group(3)))\n except ValueError:\n pass\n if event_date is not None:\n break\n \n if event_date is not None and event_date > cutoff_date:\n mendez_events_after_cutoff.append(e)\n else:\n mendez_events_on_or_before.append(e)\n \n if mendez_events_after_cutoff:\n summaries = [e.get('summary', e.get('title', '(no summary)')) for e in mendez_events_after_cutoff]\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'Found {len(mendez_events_after_cutoff)} calendar event(s) after 2026-04-10 mentioning Mendez (should not exist): {summaries}'\n }\n \n info_parts = [f'No calendar events after 2026-04-10 mentioning Mendez found among {len(all_events)} total event(s).']\n if mendez_events_on_or_before:\n summaries = [e.get('summary', e.get('title', '(no summary)')) for e in mendez_events_on_or_before]\n info_parts.append(f'{len(mendez_events_on_or_before)} Mendez event(s) on or before cutoff date (allowed): {summaries}')\n \n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': ' '.join(info_parts)\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "bb431a04-6f3a-4f8b-81e1-0158cd778eb5",
|
||||
"sort_order": 15,
|
||||
"rubric_text": "In `mailbox.json`, email with id=6 must have is_read=true and folder='Onboarding'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n email_6 = None\n for email in data.get('emails', []):\n if str(email.get('email_id', '')) == '6':\n email_6 = email\n break\n \n if email_6 is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Email with id=6 not found'}\n \n issues = []\n if not email_6.get('is_read', False):\n issues.append('Email id=6 is_read is not True')\n folder = str(email_6.get('folder', '')).lower()\n if 'onboarding' not in folder:\n issues.append(f'Email id=6 folder is \"{email_6.get(\"folder\")}\" (expected \"Onboarding\")')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Email id=6 is read and in Onboarding folder'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "70a14339-b78d-46f3-bf07-806c66effa1b",
|
||||
"sort_order": 16,
|
||||
"rubric_text": "In `mailbox.json`, a sent or forwarded email to d.okonkwo@crestwood.edu with subject containing 'Offer' and 'Academic Advisor' must exist, and email id=9 must have is_read=true and folder='General'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n with open(mailbox_path) as f:\n data = json.load(f)\n\n emails = data.get('emails', [])\n issues = []\n\n # Check sent/forwarded email to d.okonkwo@crestwood.edu with correct subject\n okonkwo_found = False\n for e in emails:\n to_addr = str(e.get('to_addr', '')).lower()\n subj = str(e.get('subject', '')).lower()\n folder = e.get('folder', '').lower()\n if 'd.okonkwo@crestwood.edu' in to_addr and folder in ('sent',) and 'offer' in subj and 'academic advisor' in subj:\n okonkwo_found = True\n break\n if not okonkwo_found:\n issues.append('No sent email to d.okonkwo@crestwood.edu with subject containing \"Offer\" and \"Academic Advisor\" found')\n\n # Check email id=9\n email_9 = next((e for e in emails if str(e.get('email_id', '')) == '9'), None)\n if email_9 is None:\n issues.append('Email id=9 not found')\n else:\n if not email_9.get('is_read', False):\n issues.append('Email id=9 is_read is not True')\n folder = str(email_9.get('folder', '')).lower()\n if 'general' not in folder:\n issues.append(f'Email id=9 folder is \"{email_9.get(\"folder\")}\" (expected \"General\")')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'Sent email to d.okonkwo@crestwood.edu with correct subject found; email id=9 is read and in General folder'}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "69905c2a-4226-433a-99eb-840ab7e7e447",
|
||||
"sort_order": 17,
|
||||
"rubric_text": "In `slack_data.json`, channel `#hr-escalations` must contain at least one message.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n issues = []\n\n if slack_path.exists():\n with open(slack_path) as f:\n slack_data = json.load(f)\n\n channels = slack_data.get('channels', {})\n escalations_id = None\n for cid, cinfo in channels.items():\n name = cinfo.get('name', '').lower()\n if name == 'hr-escalations':\n escalations_id = cid\n\n if escalations_id:\n esc_msgs = slack_data.get('messages', {}).get(escalations_id, [])\n if len(esc_msgs) == 0:\n issues.append('#hr-escalations has no messages (expected at least one)')\n else:\n issues.append('#hr-escalations channel not found in Slack')\n else:\n issues.append('slack_data.json not found')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': '#hr-escalations has at least one message'}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4d1ca98e-f0a8-460a-be73-21d822494f8c",
|
||||
"sort_order": 18,
|
||||
"rubric_text": "In `jira_state.json`, no issue with summary containing 'OFFBOARD-Navarro' must exist.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n jira_path = Path(external_services_path) / 'jira_state.json'\n issues = []\n\n if jira_path.exists():\n with open(jira_path) as f:\n jira_data = json.load(f)\n jira_issues = jira_data.get('issues', {})\n if isinstance(jira_issues, dict):\n all_jira = list(jira_issues.values())\n elif isinstance(jira_issues, list):\n all_jira = jira_issues\n else:\n all_jira = []\n navarro_jira = [i for i in all_jira if 'offboard-navarro' in str(i.get('fields', {}).get('summary', '')).lower()]\n if navarro_jira:\n issues.append(f'Found Jira issue with OFFBOARD-Navarro (should not exist)')\n else:\n issues.append('jira_state.json not found')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'No OFFBOARD-Navarro Jira issue found'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "bc3047ba-d6b3-4eb1-8815-5e2372694341",
|
||||
"sort_order": 19,
|
||||
"rubric_text": "In `slack_data.json`, channel `#hr-offboarding` must contain no message mentioning 'Navarro'.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n\n slack_path = Path(external_services_path) / 'slack_data.json'\n issues = []\n\n if slack_path.exists():\n with open(slack_path) as f:\n slack_data = json.load(f)\n\n channels = slack_data.get('channels', {})\n offboarding_id = None\n for cid, cinfo in channels.items():\n name = cinfo.get('name', '').lower()\n if name == 'hr-offboarding':\n offboarding_id = cid\n\n if offboarding_id:\n off_msgs = slack_data.get('messages', {}).get(offboarding_id, [])\n navarro_in_offboarding = [m for m in off_msgs if 'navarro' in str(m.get('text', '')).lower()]\n if navarro_in_offboarding:\n issues.append(f'Found {len(navarro_in_offboarding)} message(s) mentioning Navarro in #hr-offboarding (should not exist)')\n else:\n issues.append('slack_data.json not found')\n\n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'No messages mentioning Navarro found in #hr-offboarding'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "11427267-bf7c-46af-956a-cce9e820952e",
|
||||
"sort_order": 20,
|
||||
"rubric_text": "In `mailbox.json`, no sent email to j.park@crestwood.edu or c.briggs@crestwood.edu mentioning 'Navarro' must exist, no sent email to c.navarro@crestwood.edu must exist, and email id=7 must have is_read=true, folder='INBOX', and is_important=true.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided'}\n \n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get('emails', [])\n sent_emails = [e for e in emails if e.get('folder', '').lower() == 'sent']\n \n issues = []\n \n # Check no sent email to j.park or c.briggs mentioning Navarro\n for email in sent_emails:\n to_addr = str(email.get('to_addr', '')).lower()\n subj = str(email.get('subject', '')).lower()\n body = str(email.get('body_text', '')).lower()\n if 'j.park@crestwood.edu' in to_addr and 'navarro' in (subj + body):\n issues.append('Sent email to j.park@crestwood.edu mentioning Navarro found (should not exist)')\n if 'c.briggs@crestwood.edu' in to_addr and 'navarro' in (subj + body):\n issues.append('Sent email to c.briggs@crestwood.edu mentioning Navarro found (should not exist)')\n if 'c.navarro@crestwood.edu' in to_addr:\n issues.append('Sent email to c.navarro@crestwood.edu found (should not exist)')\n \n # Check email id=7\n email_7 = next((e for e in emails if str(e.get('email_id', '')) == '7'), None)\n if email_7 is None:\n issues.append('Email id=7 not found')\n else:\n if not email_7.get('is_read', False):\n issues.append('Email id=7 is_read is not True')\n folder = str(email_7.get('folder', '')).lower()\n if 'inbox' not in folder:\n issues.append(f'Email id=7 folder is \"{email_7.get(\"folder\")}\" (expected INBOX)')\n if not email_7.get('is_important', False):\n issues.append('Email id=7 is_important is not True')\n \n if issues:\n return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n return {'pass': True, 'score': 1.0, 'feedback': 'No Navarro-related sent emails found; email id=7 is read, in INBOX, and marked important'}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user