DerekSurge
52 lines
31 KiB
JSON
52 lines
31 KiB
JSON
[
|
||
{
|
||
"id": "22a5b683-cce3-4e6a-9561-2df413d6abd3",
|
||
"sort_order": 0,
|
||
"rubric_text": "In `mailbox.json`, there must be exactly one sent email that is a reply to email 11 (from rosa.gutierrez@ridgelinegear.com), sent to rosa.gutierrez@ridgelinegear.com, containing substantive language from both an escalation acknowledgment template and a payroll handoff template that references Teresa Vang by name. Each topic has a visual break consisting of a blank line and a relevant topic heading phrased similarly to \"Regarding your bereavement leave request:\" or \"Regarding your PTO question:\".",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\n# ---------------------------------------------------------------------------\n# Distinctive phrases extracted from each template. The model is expected to\n# adapt the template (fill placeholders, adjust wording), so we check for\n# *signature phrases* that would only appear if the template was actually used.\n# We require >= THRESHOLD matches per template rather than just 1 generic word.\n# ---------------------------------------------------------------------------\n\n# Template 31 – Escalation Acknowledgment (Pending Review)\n# Key phrases a model would reproduce if it used this template:\nTEMPLATE_31_PHRASES = [\n \"escalated it for review\",\n \"escalat\", # any form: escalated, escalation\n \"not able to confirm approval\",\n \"not able to confirm\",\n \"next steps, or a timeline\",\n \"you will hear from me\",\n \"a decision has been made\",\n \"do not interpret this message as approval\",\n \"confirmation of any specific leave dates\",\n \"pending review\",\n]\nTEMPLATE_31_THRESHOLD = 2 # must match at least 2 of the above\n\n# Template 16 – Payroll Handoff (Employee)\n# Key phrases a model would reproduce if it used this template:\nTEMPLATE_16_PHRASES = [\n \"payroll lead\",\n \"teresa vang\",\n \"forwarded your question to teresa\",\n \"she will respond to you directly\",\n \"respond to you directly\",\n \"1-2 business days\",\n \"1–2 business days\", # en-dash variant\n \"teresa.vang@ridgelinegear.com\",\n \"questions about the leave process itself\",\n \"as opposed to pay or benefits\",\n \"pay, benefits, deductions\",\n]\nTEMPLATE_16_THRESHOLD = 2 # must match at least 2 of the above\n\n\ndef _count_phrase_hits(body_lower: str, phrases: list[str]) -> tuple[int, list[str]]:\n \"\"\"Return (hit_count, list_of_matched_phrases).\"\"\"\n matched = [p for p in phrases if p in body_lower]\n return len(matched), matched\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n\n with open(mailbox_path) as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n\n # ── Locate email 11 ──────────────────────────────────────────────────\n email_11 = None\n for e in emails:\n if str(e.get(\"email_id\", \"\")) == \"11\":\n email_11 = e\n break\n\n if email_11 is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"Could not find email 11 in mailbox.json\"}\n\n email_11_message_id = email_11.get(\"message_id\", \"\")\n\n # ── Find sent replies to Rosa that are replies to email 11 ────────────\n sent_replies = []\n for e in emails:\n if e.get(\"folder\", \"\").lower() != \"sent\":\n continue\n to_addr = e.get(\"to_addr\") or e.get(\"to\") or \"\"\n if isinstance(to_addr, list):\n to_addr = \" \".join(to_addr)\n if \"rosa.gutierrez@ridgelinegear.com\" not in to_addr.lower():\n continue\n in_reply_to = e.get(\"in_reply_to\") or \"\"\n if email_11_message_id and email_11_message_id in in_reply_to:\n sent_replies.append(e)\n\n # Fallback: any sent email to Rosa (if in_reply_to was not set)\n if not sent_replies:\n for e in emails:\n if e.get(\"folder\", \"\").lower() != \"sent\":\n continue\n to_addr = e.get(\"to_addr\") or e.get(\"to\") or \"\"\n if isinstance(to_addr, list):\n to_addr = \" \".join(to_addr)\n if \"rosa.gutierrez@ridgelinegear.com\" not in to_addr.lower():\n continue\n sent_replies.append(e)\n\n if not sent_replies:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No sent reply to rosa.gutierrez@ridgelinegear.com found\"}\n\n if len(sent_replies) > 1:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Expected exactly 1 reply to Rosa Gutierrez, found {len(sent_replies)}\"}\n\n reply = sent_replies[0]\n body = reply.get(\"body_text\") or reply.get(\"body\") or \"\"\n body_lower = body.lower()\n\n issues = []\n\n # ── 1. Teresa Vang mentioned by name ─────────────────────────────────\n has_teresa = (\"teresa vang\" in body_lower\n or (\"teresa\" in body_lower and \"vang\" in body_lower))\n if not has_teresa:\n issues.append(\"Teresa Vang not mentioned by name\")\n\n # ── 2. Template 31 – Escalation Acknowledgment ───────────────────────\n t31_hits, t31_matched = _count_phrase_hits(body_lower, TEMPLATE_31_PHRASES)\n if t31_hits < TEMPLATE_31_THRESHOLD:\n issues.append(\n f\"Escalation acknowledgment (Template 31): matched only {t31_hits}/{TEMPLATE_31_THRESHOLD} \"\n f\"required phrases (matched: {t31_matched}). \"\n f\"Expected ≥{TEMPLATE_31_THRESHOLD} of: {TEMPLATE_31_PHRASES[:5]}...\"\n )\n\n # ── 3. Template 16 – Payroll Handoff ─────────────────────────────────\n t16_hits, t16_matched = _count_phrase_hits(body_lower, TEMPLATE_16_PHRASES)\n if t16_hits < TEMPLATE_16_THRESHOLD:\n issues.append(\n f\"Payroll handoff (Template 16): matched only {t16_hits}/{TEMPLATE_16_THRESHOLD} \"\n f\"required phrases (matched: {t16_matched}). \"\n f\"Expected ≥{TEMPLATE_16_THRESHOLD} of: {TEMPLATE_16_PHRASES[:5]}...\"\n )\n\n # ── 4. Visual break – topic headings with blank-line separator ───────\n bereavement_heading = re.compile(\n r'(regarding|re:|about|on the topic of|on the matter of|update on|status of)'\n r'[^\\n]*?(bereavement|leave|absence)',\n re.IGNORECASE,\n )\n pto_heading = re.compile(\n r'(regarding|re:|about|on the topic of|on the matter of|update on|status of)'\n r'[^\\n]*?(pto|payroll|pay|famli|supplement|time.off)',\n re.IGNORECASE,\n )\n\n has_bereavement_heading = bool(bereavement_heading.search(body))\n has_pto_heading = bool(pto_heading.search(body))\n has_blank_line = bool(re.search(r'\\n\\s*\\n', body))\n\n if not has_bereavement_heading:\n issues.append(\n 'No bereavement-related topic heading found '\n '(expected something like \"Regarding your bereavement leave request:\")'\n )\n if not has_pto_heading:\n issues.append(\n 'No PTO/payroll-related topic heading found '\n '(expected something like \"Regarding your PTO question:\")'\n )\n if not has_blank_line and (has_bereavement_heading or has_pto_heading):\n issues.append(\"No visual break (blank line) found between topics\")\n\n # ── Result ───────────────────────────────────────────────────────────\n if issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Reply found but missing: {'; '.join(issues)}. Body preview: {body[:500]}\",\n }\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": (\n \"Found exactly one sent reply to Rosa Gutierrez with: \"\n f\"Template 31 phrases ({t31_matched}), \"\n f\"Template 16 phrases ({t16_matched}), \"\n \"Teresa Vang mentioned by name, \"\n \"bereavement topic heading present, \"\n \"PTO/payroll topic heading present, \"\n \"visual breaks (blank lines) present.\"\n ),\n }",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "e01d14aa-f4d4-4d0f-9a39-cc6346aae71a",
|
||
"sort_order": 1,
|
||
"rubric_text": "In `mailbox.json`, Rosa Gutierrez's email must NOT be forwarded to teresa.vang@ridgelinegear.com, and no manager notification email must be sent to marcus.evans@ridgelinegear.com.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n sent_to_teresa = []\n sent_to_marcus = []\n \n for e in emails:\n if e.get(\"folder\", \"\").lower() != \"sent\":\n continue\n to_addr = (e.get(\"to_addr\") or \"\").lower()\n cc_addr = (e.get(\"cc_addr\") or \"\").lower()\n all_recipients = to_addr + \" \" + cc_addr\n \n if \"teresa.vang@ridgelinegear.com\" in all_recipients:\n sent_to_teresa.append(e)\n if \"marcus.evans@ridgelinegear.com\" in all_recipients:\n sent_to_marcus.append(e)\n \n issues = []\n if sent_to_teresa:\n issues.append(f\"Found {len(sent_to_teresa)} sent email(s) to teresa.vang@ridgelinegear.com (should not be forwarded)\")\n if sent_to_marcus:\n issues.append(f\"Found {len(sent_to_marcus)} sent email(s) to marcus.evans@ridgelinegear.com (no manager notification should be sent)\")\n \n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Confirmed no emails sent to teresa.vang@ridgelinegear.com or marcus.evans@ridgelinegear.com\"}",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "c00541bc-b57a-49ed-b06d-057405c12b32",
|
||
"sort_order": 2,
|
||
"rubric_text": "In `slack_data.json`, the #benefits-team channel must have exactly 7 posts total, with the most recent post containing the exact content 'BEREAVEMENT ESCALATION: Rosa Gutierrez, Bereavement, 04/20/2026, 04/24/2026'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find benefits-team channel\n benefits_channel_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower() == \"benefits-team\":\n benefits_channel_id = cid\n break\n \n if benefits_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"#benefits-team channel not found in slack_data.json\"}\n \n channel_messages = messages.get(benefits_channel_id, [])\n \n msg_count = len(channel_messages)\n \n if msg_count != 7:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"#benefits-team has {msg_count} messages, expected 7\"}\n \n # Find the most recent message (highest timestamp)\n sorted_msgs = sorted(channel_messages, key=lambda m: float(m.get(\"ts\", 0)), reverse=True)\n most_recent = sorted_msgs[0] if sorted_msgs else None\n \n if most_recent is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #benefits-team\"}\n \n recent_text = (most_recent.get(\"text\") or \"\").strip()\n expected_exact = \"BEREAVEMENT ESCALATION: Rosa Gutierrez, Bereavement, 04/20/2026, 04/24/2026\"\n \n # Full credit: text begins with the expected phrase (trailing punctuation/commentary allowed)\n if recent_text.startswith(expected_exact):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"#benefits-team has 7 messages, most recent begins with the expected phrase: '{expected_exact}'\"}\n \n # Partial credit: correct content but wrong date format\n recent_lower = recent_text.lower()\n has_header = \"bereavement escalation\" in recent_lower\n has_name = \"rosa gutierrez\" in recent_lower\n has_bereavement = recent_lower.count(\"bereavement\") >= 2 # once in header, once as leave type\n \n # Accept any recognizable rendering of April 20, 2026 and April 24, 2026\n date_start_patterns = [\"04/20/2026\", \"4/20/2026\", \"2026-04-20\", \"april 20, 2026\", \"april 20 2026\", \"04-20-2026\", \"20 april 2026\"]\n date_end_patterns = [\"04/24/2026\", \"4/24/2026\", \"2026-04-24\", \"april 24, 2026\", \"april 24 2026\", \"04-24-2026\", \"24 april 2026\"]\n \n has_start_date = any(p in recent_lower for p in date_start_patterns)\n has_end_date = any(p in recent_lower for p in date_end_patterns)\n \n if has_header and has_name and has_bereavement and has_start_date and has_end_date:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"#benefits-team has 7 messages, most recent has correct content but wrong date format (SOP §3.1 requires MM/DD/YYYY). Expected exact: '{expected_exact}'. Actual: '{recent_text[:300]}'\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Most recent #benefits-team post must begin with '{expected_exact}'. Actual text: '{recent_text[:500]}'\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "0475cf23-46b8-416f-86d4-1fa59591a68e",
|
||
"sort_order": 3,
|
||
"rubric_text": "In `slack_data.json`, the #payroll-coordination channel must have exactly 7 posts total, with the most recent post beginning with the exact phrase 'PAYROLL QUESTION: Rosa Gutierrez, ' and ending with a question about PTO to supplement FAMLI payments.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find payroll-coordination channel\n payroll_channel_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower() == \"payroll-coordination\":\n payroll_channel_id = cid\n break\n \n if payroll_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"#payroll-coordination channel not found in slack_data.json\"}\n \n channel_messages = messages.get(payroll_channel_id, [])\n msg_count = len(channel_messages)\n \n if msg_count != 7:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"#payroll-coordination has {msg_count} messages, expected 7\"}\n \n # Find the most recent message\n sorted_msgs = sorted(channel_messages, key=lambda m: float(m.get(\"ts\", 0)), reverse=True)\n most_recent = sorted_msgs[0] if sorted_msgs else None\n \n if most_recent is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #payroll-coordination\"}\n \n recent_text = (most_recent.get(\"text\") or \"\").strip()\n recent_text_lower = recent_text.lower()\n \n failures = []\n \n # Check prefix\n if not recent_text_lower.startswith(\"payroll question: rosa gutierrez\"):\n failures.append(f\"Most recent post does not begin with 'PAYROLL QUESTION: Rosa Gutierrez'. Actual start: '{recent_text[:120]}'\")\n \n # Check for PTO content\n if \"pto\" not in recent_text_lower:\n failures.append(\"'PTO' not found in most recent post\")\n \n # Check for FAMLI content\n if \"famli\" not in recent_text_lower:\n failures.append(\"'FAMLI' not found in most recent post\")\n \n # Check that the message conveys the concept of using PTO together with / on top of FAMLI.\n # Accept any reasonable phrasing.\n combo_terms = [\n \"supplement\", \"on top of\", \"alongside\", \"in addition to\",\n \"at the same time\", \"together with\", \"combine\", \"combined\",\n \"while on\", \"pto+famli\", \"famli+pto\", \"pto + famli\", \"famli + pto\",\n \"along with\", \"concurrent\", \"stack\", \"use pto\", \"draw pto\",\n \"add pto\", \"apply pto\", \"use my pto\", \"use her pto\",\n \"make up the difference\", \"offset\", \"top up\", \"top-up\",\n \"also receives\", \"also receive\", \"in conjunction\",\n \"both pto and famli\", \"pto and famli\", \"famli and pto\",\n ]\n combo_found = any(term in recent_text_lower for term in combo_terms)\n if not combo_found:\n failures.append(\"Post does not convey the concept of using PTO together with FAMLI\")\n \n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Most recent #payroll-coordination post issues: {'; '.join(failures)}. Full text: {recent_text[:500]}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"#payroll-coordination has 7 messages, most recent begins with 'PAYROLL QUESTION: Rosa Gutierrez' and conveys a question about using PTO with FAMLI. Text: {recent_text[:300]}\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "7241650b-e2e0-432b-91c7-29f86ea4ddc9",
|
||
"sort_order": 4,
|
||
"rubric_text": "In `slack_data.json`, the #payroll-coordination channel must NOT contain any 'LEAVE PAYROLL NOTIFICATION' post related to Rosa Gutierrez for bereavement.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find payroll-coordination channel\n payroll_channel_id = None\n for cid, cdata in channels.items():\n if cdata.get(\"name\", \"\").lower() == \"payroll-coordination\":\n payroll_channel_id = cid\n break\n \n if payroll_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"#payroll-coordination channel not found in slack_data.json\"}\n \n channel_messages = messages.get(payroll_channel_id, [])\n \n # Look for any LEAVE PAYROLL NOTIFICATION for Rosa Gutierrez bereavement\n problematic_msgs = []\n for msg in channel_messages:\n text = (msg.get(\"text\") or \"\").lower()\n if \"leave payroll notification\" in text and \"rosa gutierrez\" in text and \"bereavement\" in text:\n problematic_msgs.append(msg.get(\"text\", \"\")[:200])\n \n if problematic_msgs:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(problematic_msgs)} LEAVE PAYROLL NOTIFICATION message(s) for Rosa Gutierrez bereavement in #payroll-coordination (should not exist): {problematic_msgs[0]}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No LEAVE PAYROLL NOTIFICATION for Rosa Gutierrez bereavement found in #payroll-coordination\"}",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "55468c98-2909-42a3-8a51-bf5ec83a2b1a",
|
||
"sort_order": 5,
|
||
"rubric_text": "In `leave_tracker.xlsx` (or equivalent spreadsheet), there must be exactly 47 cases, no new bereavement case for Rosa Gutierrez must be opened, and LC-2026-0089 (Rosa Gutierrez's FMLA + FAMLI case) must remain unchanged with its existing values.",
|
||
"verifier_code": "from pathlib import Path\n\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n workspace = Path(workspace_path)\n xlsx_files = list(workspace.glob(\"*.xlsx\")) + list(workspace.glob(\"*.xls\"))\n\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Excel file found in workspace\"}\n\n # --- FIX: match the exact filename \"leave_tracker\" rather than\n # substring \"leave\" or \"tracker\", which incorrectly matched\n # hfwa_balance_tracker.xlsx before leave_tracker.xlsx. ---\n leave_tracker = None\n for f in xlsx_files:\n if f.stem.lower() == \"leave_tracker\":\n leave_tracker = f\n break\n\n # Fallback: accept any file whose stem starts with \"leave_tracker\"\n if leave_tracker is None:\n for f in xlsx_files:\n if f.stem.lower().startswith(\"leave_tracker\"):\n leave_tracker = f\n break\n\n if leave_tracker is None:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": (\n \"leave_tracker.xlsx not found in workspace. \"\n f\"Files present: {[f.name for f in xlsx_files]}\"\n ),\n }\n\n try:\n wb = openpyxl.load_workbook(leave_tracker, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {leave_tracker.name}: {e}\"}\n\n ws = wb.active\n\n # Collect data rows (skip header and fully-blank rows)\n data_rows = []\n header_row = None\n for row in ws.iter_rows(values_only=True):\n if all(cell is None for cell in row):\n continue\n if header_row is None:\n header_row = [str(c).strip().lower() if c is not None else \"\" for c in row]\n continue\n data_rows.append(row)\n\n case_count = len(data_rows)\n\n if case_count != 47:\n return {\n \"pass\": False,\n \"score\": 0.5,\n \"feedback\": f\"Expected 47 cases in {leave_tracker.name}, found {case_count}\",\n }\n\n # Locate LC-2026-0089\n lc_row = None\n for row in data_rows:\n row_str = [str(c).strip() if c is not None else \"\" for c in row]\n if any(\"lc-2026-0089\" in s.lower() for s in row_str):\n lc_row = row_str\n break\n\n if lc_row is None:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": \"LC-2026-0089 not found in leave_tracker\"}\n\n row_text = \" \".join(lc_row).lower()\n\n checks = [\n (\"rosa gutierrez\" in row_text, \"Rosa Gutierrez name not in LC-2026-0089 row\"),\n (\"fmla\" in row_text and \"famli\" in row_text, \"FMLA + FAMLI not in LC-2026-0089 row\"),\n (\"intermittent\" in row_text, \"Intermittent leave type not in LC-2026-0089 row\"),\n (\"open\" in row_text, \"Status not 'Open' for LC-2026-0089\"),\n ]\n\n failures = [msg for passed, msg in checks if not passed]\n if failures:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"LC-2026-0089 row issues: {'; '.join(failures)}\"}\n\n # Verify no new bereavement case was opened for Rosa Gutierrez\n bereavement_rosa_cases = []\n for row in data_rows:\n row_str = [str(c).strip().lower() if c is not None else \"\" for c in row]\n row_text_check = \" \".join(row_str)\n if (\n \"rosa gutierrez\" in row_text_check\n and \"bereavement\" in row_text_check\n and \"lc-2026-0089\" not in row_text_check\n ):\n bereavement_rosa_cases.append(row_str)\n\n if bereavement_rosa_cases:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(bereavement_rosa_cases)} unexpected bereavement case(s) for Rosa Gutierrez\",\n }\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": (\n f\"{leave_tracker.name} has exactly 47 cases, LC-2026-0089 found with correct values, \"\n \"no new bereavement case for Rosa Gutierrez\"\n ),\n }",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "0ca61599-e6ee-46b7-92cc-77d81d33a2c1",
|
||
"sort_order": 6,
|
||
"rubric_text": "In `mailbox.json`, no actions (reply, forward, mark) must be taken on emails 10 and 12 - no sent emails related to either email.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n # Find emails 10 and 12 to get their metadata\n target_emails = {}\n for e in emails:\n eid = str(e.get(\"email_id\", \"\"))\n if eid in (\"10\", \"12\"):\n target_emails[eid] = e\n \n # Collect message_ids, subjects, from addresses for emails 10 and 12\n target_message_ids = set()\n target_subjects = set()\n target_from_addrs = set()\n target_to_addrs = set()\n for eid, e in target_emails.items():\n mid = e.get(\"message_id\", \"\")\n if mid:\n target_message_ids.add(mid)\n subj = (e.get(\"subject\") or \"\").lower()\n if subj:\n # Strip common prefixes for matching\n clean_subj = subj\n for prefix in [\"re: \", \"fwd: \", \"fw: \"]:\n while clean_subj.startswith(prefix):\n clean_subj = clean_subj[len(prefix):]\n target_subjects.add(clean_subj)\n from_addr = (e.get(\"from_addr\") or \"\").lower().strip()\n if from_addr:\n target_from_addrs.add(from_addr)\n to_addr = (e.get(\"to_addr\") or \"\").lower().strip()\n if to_addr:\n target_to_addrs.add(to_addr)\n \n # Check for any sent emails that are replies/forwards related to emails 10 or 12\n problematic_sent = []\n for e in emails:\n if e.get(\"folder\", \"\").lower() != \"sent\":\n continue\n \n to_addr = (e.get(\"to_addr\") or \"\").lower()\n body = (e.get(\"body_text\") or \"\").lower()\n subject = (e.get(\"subject\") or \"\").lower()\n in_reply_to = e.get(\"in_reply_to\") or \"\"\n \n # Check if this sent email is related to email 10 or 12\n is_related = False\n reasons = []\n \n # Check in_reply_to against target message_ids\n if in_reply_to:\n for mid in target_message_ids:\n if mid in in_reply_to:\n is_related = True\n reasons.append(f\"in_reply_to matches message_id {mid}\")\n \n # Check if sent to original sender of emails 10 or 12\n # AND subject matches (to avoid false positives)\n clean_subject = subject\n for prefix in [\"re: \", \"fwd: \", \"fw: \"]:\n while clean_subject.startswith(prefix):\n clean_subject = clean_subject[len(prefix):]\n \n if clean_subject in target_subjects:\n # Subject matches - check if to_addr relates to the original email participants\n for addr in target_from_addrs:\n if addr in to_addr:\n is_related = True\n reasons.append(f\"to_addr contains {addr} and subject matches\")\n for addr in target_to_addrs:\n if addr in to_addr:\n is_related = True\n reasons.append(f\"to_addr contains {addr} and subject matches\")\n \n if is_related:\n problematic_sent.append(f\"To: {e.get('to_addr')}, Subject: {e.get('subject')} (reasons: {'; '.join(reasons)})\")\n \n # Also check if emails 10 or 12 have been marked/flagged (if such fields exist)\n marking_issues = []\n for eid, e in target_emails.items():\n # Check common marking fields\n if e.get(\"is_read\") is True and e.get(\"was_read\") is False:\n marking_issues.append(f\"Email {eid} was marked as read\")\n if e.get(\"flagged\") or e.get(\"starred\"):\n marking_issues.append(f\"Email {eid} was flagged/starred\")\n if e.get(\"labels_added\") or e.get(\"tags_added\"):\n marking_issues.append(f\"Email {eid} had labels/tags added\")\n \n issues = []\n if problematic_sent:\n issues.append(f\"Found {len(problematic_sent)} sent email(s) related to emails 10/12: {'; '.join(problematic_sent)}\")\n if marking_issues:\n issues.append(f\"Found marking actions on emails 10/12: {'; '.join(marking_issues)}\")\n \n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Actions were taken on emails 10 and/or 12 (should have no actions): \" + \" | \".join(issues)}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails or actions related to emails 10 and 12 found - correct, no actions should be taken on these emails.\"}",
|
||
"criterion_type": "incorrect_behavior"
|
||
}
|
||
]
|