150 lines
75 KiB
JSON
150 lines
75 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "8ad4738e-17f0-444e-83a1-b819755e7157",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In mailbox.json, there must be a sent email to j.park@crestwood.edu containing the multi-topic response with both the Employee Relations forwarding notice and the Payroll Liaison (Calvin Briggs) forwarding notice, including the phrases 'Since your message concerns multiple topics' and 'Concerning your next matter'",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n park_emails = [e for e in sent_emails if \"j.park@crestwood.edu\" in (e.get(\"to_addr\", \"\") or \"\")]\n \n if not park_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to j.park@crestwood.edu\"}\n \n required_phrases = [\n \"since your message concerns multiple topics\",\n \"concerning your next matter\",\n \"employee relations\",\n \"calvin briggs\",\n \"c.briggs@crestwood.edu\",\n \"2 business days\",\n \"payroll\"\n ]\n \n for email in park_emails:\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n missing = [p for p in required_phrases if p.lower() not in body]\n if not missing:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found qualifying sent email to j.park@crestwood.edu with all required content\"}\n \n # Check best match\n best_email = max(park_emails, key=lambda e: sum(1 for p in required_phrases if p.lower() in (e.get(\"body_text\", \"\") or \"\").lower()))\n body = (best_email.get(\"body_text\", \"\") or \"\").lower()\n missing = [p for p in required_phrases if p.lower() not in body]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Sent email to j.park found but missing required phrases: {missing}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "a9dc3bc3-d1e7-4fab-91b3-a6848a52f086",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In mailbox.json, the email 'Getting Fed Up' from j.park@crestwood.edu must be forwarded to both m.voss@crestwood.edu and c.briggs@crestwood.edu. The forwarded email may NOT contain any additional commentary, such as highlighting the content in the original email; they must only forward the email content.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if (e.get(\"folder\", \"\") or \"\").lower() == \"sent\"]\n \n # Find original 'Getting Fed Up' email from j.park\n original = None\n for e in emails:\n subj = (e.get(\"subject\", \"\") or \"\").lower()\n frm = (e.get(\"from_addr\", \"\") or \"\").lower()\n if \"getting fed up\" in subj and \"j.park@crestwood.edu\" in frm:\n original = e\n break\n \n if not original:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find original 'Getting Fed Up' email from j.park@crestwood.edu in mailbox\"}\n \n original_body = (original.get(\"body_text\", \"\") or \"\").strip()\n \n def is_forward_of_getting_fed_up(e):\n subj = (e.get(\"subject\", \"\") or \"\").lower()\n body = (e.get(\"body_text\", \"\") or \"\").lower()\n # Check subject or body references the original\n return \"getting fed up\" in subj or \"getting fed up\" in body or (original_body and original_body.lower()[:60] in body)\n \n def recipient_match(e, target_email):\n \"\"\"Check if target_email is in to, cc, or bcc fields.\"\"\"\n target = target_email.lower()\n to_field = (e.get(\"to_addr\", \"\") or \"\").lower()\n cc_field = (e.get(\"cc_addr\", \"\") or \"\").lower()\n bcc_field = (e.get(\"bcc_addr\", \"\") or \"\").lower()\n combined = to_field + \" \" + cc_field + \" \" + bcc_field\n return target in combined\n \n def has_no_additional_commentary(e):\n \"\"\"Check that the forwarded email doesn't contain additional commentary.\n The body should essentially be the original content,\n possibly with forward headers (From:, To:, Subject:, Date:, etc.).\n The agent should not add substantial new text beyond\n standard forwarding boilerplate.\"\"\"\n body = (e.get(\"body_text\", \"\") or \"\").strip()\n if not body:\n return False, \"Empty body\"\n \n # Remove common forwarding headers/markers\n cleaned = body\n # Remove forwarding delimiters\n cleaned = re.sub(r'[-=]{3,}\\s*(Forwarded|Original)\\s*(message|email)?\\s*[-=]{3,}', '', cleaned, flags=re.IGNORECASE)\n cleaned = re.sub(r'(Begin|Start)\\s+forwarded\\s+message', '', cleaned, flags=re.IGNORECASE)\n # Remove header lines that are typical in forwards\n cleaned = re.sub(r'^\\s*(From|To|Cc|Bcc|Date|Subject|Sent):.*$', '', cleaned, flags=re.IGNORECASE | re.MULTILINE)\n # Remove \"Fwd:\" or \"Fw:\" prefix artifacts\n cleaned = re.sub(r'\\b(Fwd?|Fw):\\s*', '', cleaned, flags=re.IGNORECASE)\n # Remove common short forwarding notes like \"See below\", \"FYI\", etc. - these are borderline\n # But the rubric says NO additional commentary, so we should flag these\n \n cleaned = cleaned.strip()\n \n if not original_body:\n return True, \"Original body empty, accepting\"\n \n orig_lower = original_body.lower().strip()\n body_lower = body.lower().strip()\n \n # Look for original content snippet in the body\n orig_snippet = orig_lower[:80] if len(orig_lower) > 80 else orig_lower\n idx = body_lower.find(orig_snippet)\n \n if idx < 0:\n # Try shorter snippet\n orig_snippet = orig_lower[:40] if len(orig_lower) > 40 else orig_lower\n idx = body_lower
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "4ee5a809-b91b-4f27-92a8-48ab48465cc7",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In mailbox.json, the email 'Getting Fed Up' from j.park@crestwood.edu must be in the inbox folder and flagged as important",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n park_email = None\n for e in emails:\n subj = (e.get(\"subject\", \"\") or \"\").lower()\n frm = (e.get(\"from_addr\", \"\") or \"\").lower()\n if \"getting fed up\" in subj and \"j.park@crestwood.edu\" in frm:\n park_email = e\n break\n \n if park_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email 'Getting Fed Up' from j.park@crestwood.edu not found\"}\n \n folder = (park_email.get(\"folder\", \"\") or \"\").lower()\n is_important = park_email.get(\"is_important\", False)\n \n issues = []\n if \"inbox\" not in folder:\n issues.append(f\"Email is in folder '{folder}', expected inbox\")\n if not is_important:\n issues.append(\"Email is not flagged as important\")\n \n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email 'Getting Fed Up' is in inbox and flagged as important\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "53a45309-a89a-4269-b0a3-90b9a53315d6",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "In jira_state.json, Jira tickets with names 'ER-Park-04092026' and 'BENEFITS-Park-04092026' must exist",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check jira_state.json\"}\n \n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n \n with open(jira_path) as f:\n data = json.load(f)\n \n required_names = [\"ER-Park-04092026\", \"BENEFITS-Park-04092026\"]\n \n # Collect all possible identifying strings from the Jira data\n # We look at keys, ids, names, summaries, titles — anything that could match\n all_identifiers = set()\n \n def extract_strings_from_issue(issue):\n \"\"\"Extract all string values from an issue dict that could be a ticket name/key.\"\"\"\n if not isinstance(issue, dict):\n return\n for k, v in issue.items():\n if isinstance(v, str):\n all_identifiers.add(v.strip())\n elif isinstance(v, dict):\n # Recurse one level for nested fields like 'fields'\n for kk, vv in v.items():\n if isinstance(vv, str):\n all_identifiers.add(vv.strip())\n \n if isinstance(data, dict):\n # Top-level keys might be ticket names\n all_identifiers.update(data.keys())\n \n # Check for 'issues' key\n if \"issues\" in data:\n issues = data[\"issues\"]\n if isinstance(issues, dict):\n all_identifiers.update(issues.keys())\n for v in issues.values():\n extract_strings_from_issue(v)\n elif isinstance(issues, list):\n for issue in issues:\n extract_strings_from_issue(issue)\n \n # Also check if top-level values are issue dicts\n for key, val in data.items():\n if isinstance(val, dict):\n extract_strings_from_issue(val)\n elif isinstance(val, list):\n for item in val:\n extract_strings_from_issue(item)\n elif isinstance(data, list):\n for issue in data:\n extract_strings_from_issue(issue)\n \n # Also do a brute-force text search in the raw JSON as a fallback\n raw_text = json.dumps(data)\n \n found = []\n missing = []\n for name in required_names:\n if name in all_identifiers or name in raw_text:\n found.append(name)\n else:\n missing.append(name)\n \n score = len(found) / len(required_names)\n \n if missing:\n sample_ids = sorted(list(all_identifiers))[:30]\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Missing Jira tickets: {missing}. Found tickets so far: {found}. Sample identifiers found: {sample_ids}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found all required Jira tickets: {required_names}\"\n }\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "76c8d55a-e973-4605-a6d3-2e77d5528f0e",
|
|||
|
|
"sort_order": 4,
|
|||
|
|
"rubric_text": "In jira_state.json, Jira ticket 'LEAVE-Pham-04092026' must exist",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check jira_state.json\"}\n \n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n \n with open(jira_path) as f:\n data = json.load(f)\n \n required_key = \"LEAVE-Pham-04092026\"\n # Also accept loose variants: different separators, case-insensitive\n required_key_lower = required_key.lower()\n # Build a loose pattern: LEAVE, Pham, 04092026 (or 0409 2026, 04-09-2026, etc.)\n loose_pattern = re.compile(r'LEAVE.*Pham.*04.*09.*2026', re.IGNORECASE)\n \n all_keys = set()\n all_values_str = json.dumps(data) # for deep search\n \n def collect_keys_from_dict(d):\n \"\"\"Recursively collect all string keys and interesting string values.\"\"\"\n if isinstance(d, dict):\n for k, v in d.items():\n all_keys.add(str(k))\n # Also check common fields that might hold ticket IDs\n if isinstance(v, (str,)):\n all_keys.add(v)\n elif isinstance(v, dict):\n # Check key/id/ticket_id/name fields\n for field in ['key', 'id', 'ticket_id', 'name', 'issue_key', 'issueKey', 'ticket']:\n val = v.get(field)\n if val:\n all_keys.add(str(val))\n collect_keys_from_dict(v)\n elif isinstance(v, list):\n collect_keys_from_list(v)\n \n def collect_keys_from_list(lst):\n if isinstance(lst, list):\n for item in lst:\n if isinstance(item, dict):\n for field in ['key', 'id', 'ticket_id', 'name', 'issue_key', 'issueKey', 'ticket']:\n val = item.get(field)\n if val:\n all_keys.add(str(val))\n collect_keys_from_dict(item)\n elif isinstance(item, str):\n all_keys.add(item)\n \n if isinstance(data, dict):\n collect_keys_from_dict(data)\n elif isinstance(data, list):\n collect_keys_from_list(data)\n \n # Exact match first\n if required_key in all_keys:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket: {required_key}\"}\n \n # Case-insensitive match\n for k in all_keys:\n if k.lower() == required_key_lower:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket (case-insensitive match): {k}\"}\n \n # Loose pattern match on keys\n for k in all_keys:\n if loose_pattern.search(k):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket (loose match): {k}\"}\n \n # Deep search in serialized JSON\n if loose_pattern.search(all_values_str):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket pattern in jira_state.json (deep search)\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Missing Jira ticket: {required_key}. Found keys (sample): {sorted(list(all_keys))[:30]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "a8f4180b-f3fc-40b5-a5d0-78ee8c75c44d",
|
|||
|
|
"sort_order": 5,
|
|||
|
|
"rubric_text": "In jira_state.json, Jira ticket 'LEAVE-Pham-ADA-04092026' must exist",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check jira_state.json\"}\n \n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n \n with open(jira_path) as f:\n data = json.load(f)\n \n required_key = \"LEAVE-Pham-ADA-04092026\"\n \n # Collect all possible keys/ids/summaries from the jira state in a generous manner\n all_keys = set()\n \n def extract_keys_from_issue(issue):\n \"\"\"Extract all identifiable strings from an issue dict.\"\"\"\n if not isinstance(issue, dict):\n return\n for field in [\"key\", \"id\", \"ticket_id\", \"ticketId\", \"issue_key\", \"issueKey\", \"name\", \"summary\", \"title\"]:\n val = issue.get(field)\n if val is not None:\n all_keys.add(str(val).strip())\n \n def walk(obj):\n \"\"\"Recursively walk the JSON structure to find all potential ticket identifiers.\"\"\"\n if isinstance(obj, dict):\n # The dict keys themselves might be ticket IDs\n for k, v in obj.items():\n all_keys.add(str(k).strip())\n extract_keys_from_issue(obj) # treat the dict itself as a possible issue\n walk(v)\n elif isinstance(obj, list):\n for item in obj:\n if isinstance(item, dict):\n extract_keys_from_issue(item)\n walk(item)\n elif isinstance(obj, str):\n # Could be a bare string ticket id\n all_keys.add(obj.strip())\n \n walk(data)\n \n # Direct match\n if required_key in all_keys:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket: {required_key}\"}\n \n # Case-insensitive match\n lower_keys = {k.lower(): k for k in all_keys}\n if required_key.lower() in lower_keys:\n matched = lower_keys[required_key.lower()]\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket (case-insensitive match): {matched}\"}\n \n # Partial/fuzzy: check if the key appears as substring in any collected key\n for k in all_keys:\n if required_key in k or k in required_key:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket (substring match): {k}\"}\n \n # Also do a full-text search of the raw JSON string\n raw = jira_path.read_text()\n if required_key in raw:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket in raw JSON text: {required_key}\"}\n if required_key.lower() in raw.lower():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required Jira ticket in raw JSON text (case-insensitive): {required_key}\"}\n \n sample = sorted(all_keys)[:30]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Missing Jira ticket: {required_key}. Sample of found keys/values: {sample}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "cbede6aa-9c8c-412e-a939-a1eebc1708ab",
|
|||
|
|
"sort_order": 6,
|
|||
|
|
"rubric_text": "In slack_data.json, channel #hr-leave-requests must contain a message with exact content: 'FMLA REQUEST: Diane Pham, CU-1020, Eligible: YES, Certification Due: 04/23/2026, Jira: LEAVE-Pham-04092026'",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n expected_text = \"FMLA REQUEST: Diane Pham, CU-1020, Eligible: YES, Certification Due: 04/23/2026, Jira: LEAVE-Pham-04092026\"\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find the #hr-leave-requests channel\n target_channel_id = None\n for cid, cdata in channels.items():\n if isinstance(cdata, dict) and cdata.get(\"name\", \"\").lower() == \"hr-leave-requests\":\n target_channel_id = cid\n break\n \n if target_channel_id is None:\n # Try searching all messages across all channels\n all_msgs = []\n for cid, msgs in messages.items():\n if isinstance(msgs, list):\n all_msgs.extend(msgs)\n else:\n all_msgs = messages.get(target_channel_id, [])\n \n # Collect all message texts for debugging\n found_texts = []\n \n for msg in all_msgs:\n if isinstance(msg, dict):\n text = (msg.get(\"text\", \"\") or \"\").strip()\n found_texts.append(text)\n if text == expected_text:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exact FMLA REQUEST message in #hr-leave-requests: '{text}'\"}\n \n # Try case-insensitive and whitespace-normalized\n expected_normalized = \" \".join(expected_text.lower().split())\n for msg in all_msgs:\n if isinstance(msg, dict):\n text = \" \".join((msg.get(\"text\", \"\") or \"\").lower().split())\n if text == expected_normalized:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found FMLA REQUEST message (normalized match) in #hr-leave-requests\"}\n \n # Also accept slight date variations: 04/23 or 04/24 since the rubric says 04/23\n # but be generous - check if the core content is present\n for msg in all_msgs:\n if isinstance(msg, dict):\n text = (msg.get(\"text\", \"\") or \"\").strip()\n if \"FMLA REQUEST\" in text.upper() and \"Diane Pham\" in text and \"CU-1020\" in text and \"LEAVE-Pham-04092026\" in text:\n return {\"pass\": True, \"score\": 0.8, \"feedback\": f\"Found FMLA REQUEST message with key fields matching but not exact: '{text}'. Expected: '{expected_text}'\"}\n \n # Show what we found for debugging\n sample = found_texts[:10]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Did not find message '{expected_text}' in #hr-leave-requests. Searched {len(all_msgs)} messages. Channel ID found: {target_channel_id}. Sample messages: {sample}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "7ff41aa4-cfb5-4711-b75d-6bfbb68124b4",
|
|||
|
|
"sort_order": 7,
|
|||
|
|
"rubric_text": "In calendar_data.json, there must be an event with summary 'FMLA Certification Due – Diane Pham' (or close variant) on 04/23/2026",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found\"}\n \n with open(cal_path) as f:\n data = json.load(f)\n \n events = data.get(\"events\", {})\n \n # Rubric says 04/23/2026\n target_dates = [\"2026-04-23\", \"04/23/2026\", \"04-23-2026\", \"April 23\", \"4/23/2026\"]\n # Also accept off-by-one dates to be generous\n nearby_dates = [\"2026-04-22\", \"2026-04-24\", \"04/22/2026\", \"04/24/2026\"]\n \n all_events_info = []\n \n for event_id, event in events.items():\n if not isinstance(event, dict):\n continue\n summary = (event.get(\"summary\", \"\") or \"\").strip()\n summary_lower = summary.lower()\n start = str(event.get(\"start\", \"\") or \"\")\n \n all_events_info.append(f\" ID={event_id}, summary='{summary}', start='{start}'\")\n \n # Check if summary relates to FMLA and Diane Pham\n has_fmla = \"fmla\" in summary_lower\n has_pham = \"pham\" in summary_lower or \"diane\" in summary_lower\n \n if not (has_fmla and has_pham):\n continue\n \n # Check date - primary target 04/23/2026\n date_match_exact = any(d in start for d in target_dates)\n date_match_nearby = any(d in start for d in nearby_dates)\n \n if date_match_exact:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found calendar event '{summary}' on {start} matching FMLA Certification Due for Diane Pham on 04/23/2026\"}\n elif date_match_nearby:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found calendar event '{summary}' on {start} (close to 04/23/2026) matching FMLA Certification Due for Diane Pham\"}\n \n # If we didn't find a match, provide detailed feedback\n if all_events_info:\n events_detail = \"\\n\".join(all_events_info[:20])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No calendar event found for FMLA Certification Due for Diane Pham on 04/23/2026. Events found:\\n{events_detail}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No calendar events found at all in calendar_data.json. Expected an event with summary like 'FMLA Certification Due – Diane Pham' on 04/23/2026\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "41cae001-30b6-4411-8c7b-8f16fe004348",
|
|||
|
|
"sort_order": 8,
|
|||
|
|
"rubric_text": "In mailbox.json, there must be a sent email to d.pham@crestwood.edu with attachment fmla_medical_certification.pdf containing the FMLA review content and accommodation forwarding notice, including 'Since your message concerns multiple topics' and 'Concerning your next matter', with FMLA details (eligibility, April 23 2026 deadline, p.huang@crestwood.edu) and accommodation acknowledgment (5 business days). The format of the date stated may take any form.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n pham_emails = [e for e in sent_emails if \"d.pham@crestwood.edu\" in (e.get(\"to_addr\", \"\") or \"\").lower()]\n \n if not pham_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to d.pham@crestwood.edu\"}\n \n # Helper to check for April 23 2026 in various date formats\n def has_april_23_2026(text):\n patterns = [\n r'april\\s+23[,\\s]+2026',\n r'23\\s+april[,\\s]+2026',\n r'04[/\\-]23[/\\-]2026',\n r'4[/\\-]23[/\\-]2026',\n r'2026[/\\-]04[/\\-]23',\n r'2026[/\\-]4[/\\-]23',\n r'apr\\.?\\s+23[,\\s]+2026',\n r'23\\s+apr\\.?[,\\s]+2026',\n ]\n for pat in patterns:\n if re.search(pat, text):\n return True\n return False\n \n best_score = 0\n best_missing = []\n best_attachment_info = \"\"\n \n for email in pham_emails:\n body = (email.get(\"body_text\", \"\") or \"\")\n body_lower = body.lower()\n attachments = email.get(\"attachments\", []) or []\n \n checks = {}\n \n # Check key phrases (loosely)\n checks[\"multiple_topics\"] = \"since your message concerns multiple topics\" in body_lower or \"multiple topics\" in body_lower\n checks[\"next_matter\"] = \"concerning your next matter\" in body_lower or \"next matter\" in body_lower or \"your next\" in body_lower\n checks[\"fmla\"] = \"fmla\" in body_lower\n checks[\"april_23_2026\"] = has_april_23_2026(body_lower)\n checks[\"p_huang\"] = \"p.huang@crestwood.edu\" in body_lower\n checks[\"5_business_days\"] = \"5 business days\" in body_lower or \"five business days\" in body_lower\n checks[\"eligibility\"] = \"eligib\" in body_lower\n checks[\"medical_certification\"] = \"medical certification\" in body_lower or \"medical\" in body_lower\n checks[\"accommodation\"] = \"accommodat\" in body_lower\n \n # Check for attachment - fmla_medical_certification.pdf\n has_attachment = any(\n \"fmla_medical_certification\" in str(a).lower() or\n (\"fmla\" in str(a).lower() and (\"certification\" in str(a).lower() or \"medical\" in str(a).lower()))\n for a in attachments\n ) if attachments else False\n \n passed = sum(1 for v in checks.values() if v)\n total = len(checks)\n \n missing = [k for k, v in checks.items() if not v]\n \n if passed > best_score:\n best_score = passed\n best_missing = missing\n best_attachment_info = f\"has_attachment={has_attachment}, attachments={attachments}\"\n \n # Core required: fmla, april_23_2026, p_huang, 5_business_days, multiple_topics or next_matter\n core_pass = (checks[\"fmla\"] and checks[\"april_23_2026\"] and checks[\"p_huang\"] \n and checks[\"5_business_days\"] \n and (checks[\"multiple_topics\"] or checks[\"next_matter\"]))\n \n if core_pass and has_attachment:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found qualifying email to d.pham with all required content and FMLA attachment. Checks: {checks}\"}\n \n if core_pass:\n return {\"pass\": True, \"score\": 0.9, \"feedback\": f\"Found email to d.pham w
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "250c1cde-31b7-49f8-959c-af125c12f674",
|
|||
|
|
"sort_order": 9,
|
|||
|
|
"rubric_text": "In mailbox.json, the email 'Upcoming Surgery in June' from d.pham@crestwood.edu must be forwarded to p.huang@crestwood.edu. The forwarded email may NOT contain any additional commentary, such as highlighting the content in the original email; they must only forward the email content.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n # Find the original email from d.pham about upcoming surgery\n original_email = None\n for e in emails:\n from_addr = (e.get(\"from_addr\", \"\") or \"\").lower()\n subject = (e.get(\"subject\", \"\") or \"\").lower()\n if \"d.pham@crestwood.edu\" in from_addr and (\"upcoming surgery\" in subject or \"surgery in june\" in subject):\n original_email = e\n break\n \n if original_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find original 'Upcoming Surgery in June' email from d.pham@crestwood.edu in mailbox\"}\n \n original_body = (original_email.get(\"body_text\", \"\") or \"\").strip()\n \n # Find sent emails to p.huang\n sent_emails = [e for e in emails if (e.get(\"folder\", \"\") or \"\").lower() == \"sent\"]\n huang_emails = [e for e in sent_emails if \"p.huang@crestwood.edu\" in (e.get(\"to_addr\", \"\") or \"\").lower()]\n \n if not huang_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to p.huang@crestwood.edu\"}\n \n # Check if any look like a forward of 'Upcoming Surgery in June'\n def is_fwd_of_surgery(e):\n subj = (e.get(\"subject\", \"\") or \"\").lower()\n body = (e.get(\"body_text\", \"\") or \"\").lower()\n return (\"upcoming surgery\" in subj or \"upcoming surgery\" in body or \n \"surgery in june\" in subj or \"surgery in june\" in body)\n \n fwd_emails = [e for e in huang_emails if is_fwd_of_surgery(e)]\n \n if not fwd_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No forward of 'Upcoming Surgery in June' found to p.huang@crestwood.edu. Found {len(huang_emails)} email(s) to huang but none match.\"}\n \n # Now check that the forwarded email does NOT contain additional commentary\n # The forwarded email body should essentially be the original email content,\n # possibly with standard forwarding headers, but no extra commentary added.\n for fwd in fwd_emails:\n fwd_body = (fwd.get(\"body_text\", \"\") or \"\").strip()\n \n if not fwd_body:\n continue\n \n # Remove common forwarding headers/metadata lines that email clients add\n # These are acceptable and not \"commentary\"\n cleaned_body = fwd_body\n # Remove lines like \"---------- Forwarded message ----------\"\n cleaned_body = re.sub(r'-{2,}\\s*Forwarded\\s+message\\s*-{2,}', '', cleaned_body, flags=re.IGNORECASE)\n # Remove forwarding metadata lines (From:, Date:, Subject:, To:)\n cleaned_body = re.sub(r'^\\s*(From|Date|Subject|To|Sent|Cc)\\s*:.*$', '', cleaned_body, flags=re.IGNORECASE | re.MULTILINE)\n # Remove \"Begin forwarded message\" type headers\n cleaned_body = re.sub(r'Begin\\s+forwarded\\s+message', '', cleaned_body, flags=re.IGNORECASE)\n cleaned_body = cleaned_body.strip()\n \n # Similarly clean the original body for comparison\n cleaned_original = original_body.strip()\n \n # Check if the cleaned forwarded body is essentially the original content\n # or if there's significant extra text (commentary)\n # We'll check if the original content is contained in the forward\n if cleaned_original.lower() in cleaned_body.lower():\n # Now check for extra content beyond the original\n # Remove the original conte
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "8fdbed92-cf65-4b78-a458-172414fdcdf7",
|
|||
|
|
"sort_order": 10,
|
|||
|
|
"rubric_text": "In mailbox.json, the email 'Upcoming Surgery in June' from d.pham@crestwood.edu must be in the inbox folder and flagged as important",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n pham_email = None\n for e in emails:\n subj = (e.get(\"subject\", \"\") or \"\").lower()\n frm = (e.get(\"from_addr\", \"\") or \"\").lower()\n if (\"upcoming surgery\" in subj or \"surgery in june\" in subj) and \"d.pham@crestwood.edu\" in frm:\n pham_email = e\n break\n \n if pham_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email 'Upcoming Surgery in June' from d.pham@crestwood.edu not found\"}\n \n folder = (pham_email.get(\"folder\", \"\") or \"\").lower()\n is_important = pham_email.get(\"is_important\", False)\n \n issues = []\n if \"inbox\" not in folder:\n issues.append(f\"Email is in folder '{folder}', expected inbox\")\n if not is_important:\n issues.append(\"Email is not flagged as important\")\n \n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email 'Upcoming Surgery in June' is in inbox and flagged as important\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "71bd38c9-a88d-4690-9d8f-06ddffd71115",
|
|||
|
|
"sort_order": 11,
|
|||
|
|
"rubric_text": "In mailbox.json, there must be NO sent email to facilities@crestwood.edu",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n facilities_emails = [e for e in sent_emails if \"facilities@crestwood.edu\" in (e.get(\"to_addr\", \"\") or \"\")]\n \n if facilities_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(facilities_emails)} sent email(s) to facilities@crestwood.edu - these should not exist\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails to facilities@crestwood.edu found (correct)\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "9af8c92a-94db-4ff0-9c39-a7c6e3427e3b",
|
|||
|
|
"sort_order": 12,
|
|||
|
|
"rubric_text": "In slack_data.json, channel #hr-general must contain exactly TWO messages containing '[OUT OF SCOPE]' text",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find #hr-general channel\n target_channel_id = None\n for cid, cdata in channels.items():\n if isinstance(cdata, dict) and cdata.get(\"name\", \"\").lower() == \"hr-general\":\n target_channel_id = cid\n break\n \n if target_channel_id is None:\n # Search all channels for out of scope messages\n all_oos = []\n for cid, msgs in messages.items():\n if isinstance(msgs, list):\n for msg in msgs:\n if isinstance(msg, dict) and \"[out of scope]\" in (msg.get(\"text\", \"\") or \"\").lower():\n all_oos.append(msg)\n if len(all_oos) >= 2:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found {len(all_oos)} [OUT OF SCOPE] messages across channels (hr-general channel not found by name)\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Channel #hr-general not found and only {len(all_oos)} [OUT OF SCOPE] messages found across all channels\"}\n \n channel_msgs = messages.get(target_channel_id, [])\n oos_msgs = [m for m in channel_msgs if isinstance(m, dict) and \"[out of scope]\" in (m.get(\"text\", \"\") or \"\").lower()]\n \n if len(oos_msgs) == 2:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 2 [OUT OF SCOPE] messages in #hr-general\"}\n elif len(oos_msgs) > 2:\n return {\"pass\": True, \"score\": 0.8, \"feedback\": f\"Found {len(oos_msgs)} [OUT OF SCOPE] messages in #hr-general (expected exactly 2)\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(oos_msgs)} [OUT OF SCOPE] messages in #hr-general, expected 2\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "748ffa17-8200-43e1-a28e-01e4e1f740ba",
|
|||
|
|
"sort_order": 13,
|
|||
|
|
"rubric_text": "In mailbox.json, the email 'Re: HR office light bulb out – Main Campus rm 214' from facilities@crestwood.edu must be in the 'General' folder",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n target_email = None\n for e in emails:\n subj = (e.get(\"subject\", \"\") or \"\").lower()\n frm = (e.get(\"from_addr\", \"\") or \"\").lower()\n if (\"light bulb\" in subj or \"rm 214\" in subj or \"room 214\" in subj) and \"facilities@crestwood.edu\" in frm:\n target_email = e\n break\n \n if target_email is None:\n # Try looser search\n for e in emails:\n frm = (e.get(\"from_addr\", \"\") or \"\").lower()\n subj = (e.get(\"subject\", \"\") or \"\").lower()\n if \"facilities@crestwood.edu\" in frm and (\"light\" in subj or \"214\" in subj):\n target_email = e\n break\n \n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email 'Re: HR office light bulb out – Main Campus rm 214' from facilities@crestwood.edu not found\"}\n \n folder = (target_email.get(\"folder\", \"\") or \"\")\n if folder.lower() == \"general\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email is in 'General' folder as required\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email is in folder '{folder}', expected 'General'\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "2333bfe0-6d7b-4756-bd8f-1103a0d5836d",
|
|||
|
|
"sort_order": 14,
|
|||
|
|
"rubric_text": "In mailbox.json, there must be a sent email to p.huang@crestwood.edu with n.ashford@crestwood.edu and j.whitfield@crestwood.edu as CC recipients, containing the exact I-9 overdue notice for Tomás Salazar (CU-1026) including the work stoppage notice and escalation statement",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n huang_emails = [e for e in sent_emails if \"p.huang@crestwood.edu\" in (e.get(\"to_addr\", \"\") or \"\")]\n \n if not huang_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to p.huang@crestwood.edu\"}\n \n required_phrases = [\n \"i-9\",\n \"tom\", # covers Tomás or Tomas\n \"salazar\",\n \"cu-1026\",\n \"may not continue working\",\n \"escalated\"\n ]\n \n required_ccs = [\"n.ashford@crestwood.edu\", \"j.whitfield@crestwood.edu\"]\n \n for email in huang_emails:\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n cc = (email.get(\"cc_addr\", \"\") or \"\").lower()\n \n missing_phrases = [p for p in required_phrases if p.lower() not in body]\n missing_ccs = [c for c in required_ccs if c not in cc]\n \n if not missing_phrases and not missing_ccs:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found I-9 overdue email to p.huang with correct CC recipients and content\"}\n \n # Check best match\n best = None\n best_score = -1\n for email in huang_emails:\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n cc = (email.get(\"cc_addr\", \"\") or \"\").lower()\n score = sum(1 for p in required_phrases if p.lower() in body) + sum(1 for c in required_ccs if c in cc)\n if score > best_score:\n best_score = score\n best = email\n \n if best:\n body = (best.get(\"body_text\", \"\") or \"\").lower()\n cc = (best.get(\"cc_addr\", \"\") or \"\").lower()\n missing_phrases = [p for p in required_phrases if p.lower() not in body]\n missing_ccs = [c for c in required_ccs if c not in cc]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Best matching email missing phrases: {missing_phrases}, missing CCs: {missing_ccs}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No qualifying I-9 overdue email found to p.huang@crestwood.edu\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "04f3cd4d-a775-49cb-8bdb-9190868ebb54",
|
|||
|
|
"sort_order": 15,
|
|||
|
|
"rubric_text": "In slack_data.json, channel #hr-escalations must contain a message with exact text: 'I-9 OVERDUE: Tomás Salazar, CU-1026, Start Date: 04/02/2026, Deadline: 04/08/2026'",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n expected_text = \"I-9 OVERDUE: Tom\\u00e1s Salazar, CU-1026, Start Date: 04/02/2026, Deadline: 04/08/2026\"\n # Also accept ASCII variant\n expected_ascii = \"I-9 OVERDUE: Tomas Salazar, CU-1026, Start Date: 04/02/2026, Deadline: 04/08/2026\"\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n target_channel_id = None\n for cid, cdata in channels.items():\n if isinstance(cdata, dict) and cdata.get(\"name\", \"\").lower() == \"hr-escalations\":\n target_channel_id = cid\n break\n \n if target_channel_id is None:\n # Search all channels\n all_msgs = []\n for cid, msgs in messages.items():\n if isinstance(msgs, list):\n all_msgs.extend(msgs)\n else:\n all_msgs = messages.get(target_channel_id, [])\n \n for msg in all_msgs:\n if isinstance(msg, dict):\n text = (msg.get(\"text\", \"\") or \"\").strip()\n if text == expected_text or text == expected_ascii:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found exact I-9 OVERDUE message in #hr-escalations\"}\n \n # Normalized check\n for msg in all_msgs:\n if isinstance(msg, dict):\n text = (msg.get(\"text\", \"\") or \"\")\n normalized = \" \".join(text.lower().split())\n exp_normalized = \" \".join(expected_ascii.lower().split())\n if normalized == exp_normalized:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found I-9 OVERDUE message (normalized match) in #hr-escalations\"}\n \n # Partial match check for feedback\n key_parts = [\"i-9 overdue\", \"salazar\", \"cu-1026\", \"04/02/2026\", \"04/08/2026\"]\n for msg in all_msgs:\n if isinstance(msg, dict):\n text = (msg.get(\"text\", \"\") or \"\").lower()\n if all(p.lower() in text for p in key_parts):\n return {\"pass\": True, \"score\": 0.9, \"feedback\": f\"Found I-9 OVERDUE message with all key parts (minor formatting difference): '{msg.get('text', '')}'\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Did not find I-9 OVERDUE message for Salazar CU-1026 in #hr-escalations\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "8e620d97-f819-418c-9f15-0074bca51f49",
|
|||
|
|
"sort_order": 16,
|
|||
|
|
"rubric_text": "In mailbox.json, there must be a sent email to r.mendez@crestwood.edu containing the benefits enrollment deadline reminder with April 24, 2026 deadline, mention of p.huang@crestwood.edu, and the 30 calendar days reference. The format of the date may take any form.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n mendez_emails = [e for e in sent_emails if \"r.mendez@crestwood.edu\" in (e.get(\"to_addr\", \"\") or \"\").lower()]\n \n if not mendez_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to r.mendez@crestwood.edu\"}\n \n # Date patterns for April 24, 2026 in any format\n date_patterns = [\n r\"april\\s+24[,\\s]+2026\",\n r\"04[/-]24[/-]2026\",\n r\"2026[/-]04[/-]24\",\n r\"24[/-]04[/-]2026\",\n r\"4[/-]24[/-]2026\",\n r\"2026[/-]4[/-]24\",\n r\"24[/-]4[/-]2026\",\n r\"apr\\.?\\s+24[,\\s]+2026\",\n r\"24\\s+april[,\\s]+2026\",\n r\"24\\s+apr\\.?[,\\s]+2026\",\n r\"april\\s+24th[,\\s]+2026\",\n r\"24th\\s+(?:of\\s+)?april[,\\s]+2026\",\n r\"04[/-]24[/-]26\",\n r\"4[/-]24[/-]26\",\n r\"24[/-]04[/-]26\",\n r\"24[/-]4[/-]26\",\n ]\n \n # Check each email to r.mendez for the required content\n for email in mendez_emails:\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n full_text = body + \" \" + subject\n \n has_date = any(re.search(p, full_text) for p in date_patterns)\n has_huang = \"p.huang@crestwood.edu\" in full_text\n has_30_days = bool(re.search(r\"30\\s+calendar\\s+days?\", full_text))\n has_benefits = \"benefit\" in full_text\n has_enrollment = \"enroll\" in full_text\n \n missing = []\n if not has_date:\n missing.append(\"April 24, 2026 deadline (any format)\")\n if not has_huang:\n missing.append(\"p.huang@crestwood.edu\")\n if not has_30_days:\n missing.append(\"30 calendar days\")\n if not has_benefits:\n missing.append(\"benefits mention\")\n if not has_enrollment:\n missing.append(\"enrollment mention\")\n \n if not missing:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found qualifying benefits enrollment reminder email to r.mendez@crestwood.edu with April 24, 2026 deadline, p.huang@crestwood.edu reference, and 30 calendar days mention.\"}\n \n # Find best email for feedback\n def score_email(email):\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n full_text = body + \" \" + subject\n s = 0\n if any(re.search(p, full_text) for p in date_patterns):\n s += 1\n if \"p.huang@crestwood.edu\" in full_text:\n s += 1\n if re.search(r\"30\\s+calendar\\s+days?\", full_text):\n s += 1\n if \"benefit\" in full_text:\n s += 1\n if \"enroll\" in full_text:\n s += 1\n return s\n \n best = max(mendez_emails, key=score_email)\n body = (best.get(\"body_text\", \"\") or \"\").lower()\n subject = (best.get(\"subject\", \"\") or \"\").lower()\n full_text = body + \" \" + subject\n \n missing = []\n if not any(re.search(p, full_text) for p in date_patterns):\n missing.append(\"April 24, 2026 deadline (any format)\")\n if \"p.huang@crestwood.edu\" not in full_text:\n missing.append(\"p.huang@crestwood.edu\")\n if not re.search(r\"30\\s+calendar\\s+days?\", full_text):\n missing.append(\
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "90e8d92c-e56e-459c-88cf-288fb0035d34",
|
|||
|
|
"sort_order": 17,
|
|||
|
|
"rubric_text": "In `onboarding_checklist.xlsx`, the Notes column for Rafael Mendez must contain text mentioning April 13 (the date the reminder was sent)",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n \n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n checklist_files = [f for f in xlsx_files if \"onboarding\" in f.name.lower() and \"checklist\" in f.name.lower()]\n \n if not checklist_files:\n checklist_files = xlsx_files\n \n if not checklist_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace\"}\n \n april13_patterns = [\n r\"april\\s*13\",\n r\"apr\\.?\\s*13\",\n r\"4[/\\-]13\",\n r\"04[/\\-]13\",\n ]\n \n for f in checklist_files:\n try:\n wb = openpyxl.load_workbook(f, data_only=True)\n for sheet in wb.worksheets:\n # Find header row to identify columns\n header_row = None\n notes_col = None\n name_col = None\n \n for row in sheet.iter_rows():\n for cell in row:\n val = str(cell.value or \"\").lower()\n if \"notes\" in val and notes_col is None:\n notes_col = cell.column\n header_row = cell.row\n if (\"name\" in val or \"employee\" in val) and name_col is None:\n name_col = cell.column\n if header_row:\n break\n \n # Search for Mendez row\n for row in sheet.iter_rows():\n row_has_mendez = False\n notes_value = None\n \n for cell in row:\n cell_val = str(cell.value or \"\")\n if \"mendez\" in cell_val.lower() or \"rafael\" in cell_val.lower():\n row_has_mendez = True\n if notes_col and cell.column == notes_col:\n notes_value = cell_val\n \n if row_has_mendez and notes_value:\n notes_lower = notes_value.lower()\n for pattern in april13_patterns:\n if re.search(pattern, notes_lower, re.IGNORECASE):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found April 13 reference in Notes for Rafael Mendez: '{notes_value}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Notes for Rafael Mendez found but no April 13 reference: '{notes_value}'\"}\n \n # Fallback: search all cells for mendez + april 13 context\n for row in sheet.iter_rows():\n row_vals = [str(c.value or \"\") for c in row]\n full_row = \" \".join(row_vals)\n if \"mendez\" in full_row.lower() or \"rafael\" in full_row.lower():\n for pattern in april13_patterns:\n if re.search(pattern, full_row, re.IGNORECASE):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found April 13 reference in Rafael Mendez row: '{full_row[:200]}'\"}\n except Exception as ex:\n continue\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find April 13 reference in Notes column for Rafael Mendez in any xlsx file. Files checked: {[f.name for f in checklist_files]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "72d6314a-f04d-48b0-9ed6-fbd02f10087a",
|
|||
|
|
"sort_order": 18,
|
|||
|
|
"rubric_text": "In mailbox.json, emails 34, 35, 36 must all be marked as read (is_read: true)",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n # Find emails with IDs 34, 35, 36 - accept both int and string representations\n target_ids = {34, 35, 36}\n target_emails = []\n for e in emails:\n eid = e.get(\"email_id\", e.get(\"id\", \"\"))\n try:\n if int(eid) in target_ids:\n target_emails.append(e)\n except (ValueError, TypeError):\n continue\n \n found_ids = set()\n for e in target_emails:\n eid = e.get(\"email_id\", e.get(\"id\", \"\"))\n try:\n found_ids.add(int(eid))\n except (ValueError, TypeError):\n pass\n \n missing_ids = target_ids - found_ids\n if missing_ids:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find emails with IDs {sorted(missing_ids)} in mailbox.json. Found email IDs: {sorted(found_ids)}\"}\n \n unread = []\n for e in target_emails:\n is_read = e.get(\"is_read\", False)\n # Accept truthy values loosely\n if not is_read and is_read != \"true\":\n eid = e.get(\"email_id\", e.get(\"id\", \"\"))\n unread.append(eid)\n \n if not unread:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"All 3 target emails (34, 35, 36) are marked as read (is_read: true)\"}\n else:\n read_ids = sorted(found_ids - {int(uid) for uid in unread if uid is not None})\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Emails {unread} are not marked as read. Only emails {read_ids} out of [34, 35, 36] have is_read: true.\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776130249185",
|
|||
|
|
"sort_order": 19,
|
|||
|
|
"rubric_text": "There should not be two sent emails to p.huang that discuss \"The FMLA Medical Certification for Meghan Cross\", nor should there be two Slack messages regarding \"FMLA CERTIFICATION OVERDUE\".",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n feedback_parts = []\n email_duplicate = False\n slack_duplicate = False\n\n # --- Check sent emails for duplicates to p.huang about FMLA Medical Certification for Meghan Cross ---\n if external_services_path:\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n mailbox = json.load(f)\n emails = mailbox.get(\"emails\", [])\n matching_emails = []\n for email in emails:\n folder = (email.get(\"folder\") or \"\").lower()\n if folder != \"sent\":\n continue\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n if \"p.huang\" not in to_addr and \"phuang\" not in to_addr.replace(\".\", \"\"):\n continue\n subject = (email.get(\"subject\") or \"\").lower()\n body = (email.get(\"body_text\") or \"\").lower()\n combined = subject + \" \" + body\n if \"fmla\" in combined and \"medical certification\" in combined and \"meghan\" in combined:\n matching_emails.append(email)\n if len(matching_emails) >= 2:\n email_duplicate = True\n feedback_parts.append(\n f\"FAIL: Found {len(matching_emails)} sent emails to p.huang discussing 'The FMLA Medical Certification for Meghan Cross'. \"\n f\"Subjects: {[e.get('subject','') for e in matching_emails]}\"\n )\n else:\n feedback_parts.append(\n f\"PASS (email check): Found {len(matching_emails)} sent email(s) to p.huang about FMLA Medical Certification for Meghan Cross (0 or 1 is acceptable).\"\n )\n else:\n feedback_parts.append(\"No mailbox.json found; cannot verify email duplicates. Treating as pass for email check.\")\n else:\n feedback_parts.append(\"No external_services_path provided; cannot verify email duplicates. Treating as pass for email check.\")\n\n # --- Check Slack messages for duplicates about FMLA CERTIFICATION OVERDUE ---\n slack_matching = []\n # Check external services for slack\n if external_services_path:\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n # slack_data could have various structures; try common ones\n messages = []\n if isinstance(slack_data, dict):\n for channel, channel_data in slack_data.items():\n if isinstance(channel_data, list):\n messages.extend(channel_data)\n elif isinstance(channel_data, dict):\n msgs = channel_data.get(\"messages\", [])\n if isinstance(msgs, list):\n messages.extend(msgs)\n elif isinstance(slack_data, list):\n messages = slack_data\n\n for msg in messages:\n text = \"\"\n if isinstance(msg, dict):\n text = (msg.get(\"text\") or msg.get(\"message\") or msg.get(\"content\") or \"\").lower()\n elif isinstance(msg, str):\n text = msg.lower()\n if \"fmla\" in text and \"certification\" in text and \"overdue\" in text:\n slack_matching.append(msg)\n\n if len(slack_matching) >= 2:\n slack_duplicate = True\n feedback_parts.append(\n f\"FAIL: Found {len(slack_matching)} Slack messages regarding 'FMLA CERTIFICATION OVERDUE'.\"\n )\n else:\n feedback_parts.append(\n f\"PASS (Slack check):
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776130372291",
|
|||
|
|
"sort_order": 20,
|
|||
|
|
"rubric_text": "There should not be a message sent to `p.nair@crestwood.edu` or `t.salazar@crestwood.edu` regarding \"benefits enrollment deadline\".",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external services path provided; cannot confirm any emails were sent, so passing by default.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found; no emails to check, so passing by default.\"}\n \n try:\n with open(mailbox_path, 'r') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not parse mailbox.json: {e}. Passing by default.\"}\n \n emails = mailbox_data.get(\"emails\", [])\n \n target_addresses = [\"p.nair@crestwood.edu\", \"t.salazar@crestwood.edu\"]\n keyword = \"benefits enrollment deadline\"\n \n violating_emails = []\n \n for email in emails:\n # Only check sent emails (or drafts that may have been sent)\n folder = email.get(\"folder\", \"\").lower()\n if folder not in [\"sent\", \"drafts\"]:\n continue\n \n to_addr = email.get(\"to_addr\", \"\").lower().strip()\n subject = email.get(\"subject\", \"\").lower().strip()\n body = email.get(\"body_text\", \"\").lower().strip()\n \n # Check if the email is addressed to one of the target addresses\n addressed_to_target = False\n for target in target_addresses:\n if target.lower() in to_addr:\n addressed_to_target = True\n break\n \n if not addressed_to_target:\n continue\n \n # Check if the email is about \"benefits enrollment deadline\"\n combined_text = subject + \" \" + body\n if \"benefits\" in combined_text and (\"enrollment\" in combined_text or \"deadline\" in combined_text):\n violating_emails.append({\n \"to\": email.get(\"to_addr\", \"\"),\n \"subject\": email.get(\"subject\", \"\"),\n \"folder\": email.get(\"folder\", \"\")\n })\n \n if violating_emails:\n details = \"; \".join([f\"To: {e['to']}, Subject: {e['subject']}, Folder: {e['folder']}\" for e in violating_emails])\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violating_emails)} email(s) sent to p.nair@crestwood.edu or t.salazar@crestwood.edu regarding 'benefits enrollment deadline': [{details}]\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No emails were sent to p.nair@crestwood.edu or t.salazar@crestwood.edu regarding 'benefits enrollment deadline'.\"\n }\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
}
|
|||
|
|
]
|