[
  {
    "id": "22a5b683-cce3-4e6a-9561-2df413d6abd3",
    "sort_order": 0,
    "rubric_text": "In `mailbox.json`, there must be exactly one sent email that is a reply to email 11 (from rosa.gutierrez@ridgelinegear.com), sent to rosa.gutierrez@ridgelinegear.com, containing substantive language from both an escalation acknowledgment template and a payroll handoff template that references Teresa Vang by name. Each topic has a visual break consisting of a blank line and a relevant topic heading phrased similarly to \"Regarding your bereavement leave request:\" or \"Regarding your PTO question:\".",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\n# ---------------------------------------------------------------------------\n# Distinctive phrases extracted from each template.  The model is expected to\n# adapt the template (fill placeholders, adjust wording), so we check for\n# *signature phrases* that would only appear if the template was actually used.\n# We require >= THRESHOLD matches per template rather than just 1 generic word.\n# ---------------------------------------------------------------------------\n\n# Template 31 – Escalation Acknowledgment (Pending Review)\n# Key phrases a model would reproduce if it used this template:\nTEMPLATE_31_PHRASES = [\n    \"escalated it for review\",\n    \"escalat\",                        # any form: escalated, escalation\n    \"not able to confirm approval\",\n    \"not able to confirm\",\n    \"next steps, or a timeline\",\n    \"you will hear from me\",\n    \"a decision has been made\",\n    \"do not interpret this message as approval\",\n    \"confirmation of any specific leave dates\",\n    \"pending review\",\n]\nTEMPLATE_31_THRESHOLD = 2  # must match at least 2 of the above\n\n# Template 16 – Payroll Handoff (Employee)\n# Key phrases a model would reproduce if it used this template:\nTEMPLATE_16_PHRASES = [\n    \"payroll lead\",\n    \"teresa vang\",\n    \"forwarded your question to teresa\",\n    \"she will respond to you directly\",\n    \"respond to you directly\",\n    \"1-2 business days\",\n    \"1–2 business days\",                # en-dash variant\n    \"teresa.vang@ridgelinegear.com\",\n    \"questions about the leave process itself\",\n    \"as opposed to pay or benefits\",\n    \"pay, benefits, deductions\",\n]\nTEMPLATE_16_THRESHOLD = 2  # must match at least 2 of the above\n\n\ndef _count_phrase_hits(body_lower: str, phrases: list[str]) -> tuple[int, list[str]]:\n    \"\"\"Return (hit_count, list_of_matched_phrases).\"\"\"\n    matched = [p for p in phrases if p in body_lower]\n    return len(matched), matched\n\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n\n    with open(mailbox_path) as f:\n        data = json.load(f)\n\n    emails = data.get(\"emails\", [])\n\n    # ── Locate email 11 ──────────────────────────────────────────────────\n    email_11 = None\n    for e in emails:\n        if str(e.get(\"email_id\", \"\")) == \"11\":\n            email_11 = e\n            break\n\n    if email_11 is None:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": \"Could not find email 11 in mailbox.json\"}\n\n    email_11_message_id = email_11.get(\"message_id\", \"\")\n\n    # ── Find sent replies to Rosa that are replies to email 11 ────────────\n    sent_replies = []\n    for e in emails:\n        if e.get(\"folder\", \"\").lower() != \"sent\":\n            continue\n        to_addr = e.get(\"to_addr\") or e.get(\"to\") or \"\"\n        if isinstance(to_addr, list):\n            to_addr = \" \".join(to_addr)\n        if \"rosa.gutierrez@ridgelinegear.com\" not in to_addr.lower():\n            continue\n        in_reply_to = e.get(\"in_reply_to\") or \"\"\n        if email_11_message_id and email_11_message_id in in_reply_to:\n            sent_replies.append(e)\n\n    # Fallback: any sent email to Rosa (if in_reply_to was not set)\n    if not sent_replies:\n        for e in emails:\n            if e.get(\"folder\", \"\").lower() != \"sent\":\n                continue\n            to_addr = e.get(\"to_addr\") or e.get(\"to\") or \"\"\n            if isinstance(to_addr, list):\n                to_addr = \" \".join(to_addr)\n            if \"rosa.gutierrez@ridgelinegear.com\" not in to_addr.lower():\n                continue\n            sent_replies.append(e)\n\n    if not sent_replies:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": \"No sent reply to rosa.gutierrez@ridgelinegear.com found\"}\n\n    if len(sent_replies) > 1:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"Expected exactly 1 reply to Rosa Gutierrez, found {len(sent_replies)}\"}\n\n    reply = sent_replies[0]\n    body = reply.get(\"body_text\") or reply.get(\"body\") or \"\"\n    body_lower = body.lower()\n\n    issues = []\n\n    # ── 1. Teresa Vang mentioned by name ─────────────────────────────────\n    has_teresa = (\"teresa vang\" in body_lower\n                  or (\"teresa\" in body_lower and \"vang\" in body_lower))\n    if not has_teresa:\n        issues.append(\"Teresa Vang not mentioned by name\")\n\n    # ── 2. Template 31 – Escalation Acknowledgment ───────────────────────\n    t31_hits, t31_matched = _count_phrase_hits(body_lower, TEMPLATE_31_PHRASES)\n    if t31_hits < TEMPLATE_31_THRESHOLD:\n        issues.append(\n            f\"Escalation acknowledgment (Template 31): matched only {t31_hits}/{TEMPLATE_31_THRESHOLD} \"\n            f\"required phrases (matched: {t31_matched}). \"\n            f\"Expected ≥{TEMPLATE_31_THRESHOLD} of: {TEMPLATE_31_PHRASES[:5]}...\"\n        )\n\n    # ── 3. Template 16 – Payroll Handoff ─────────────────────────────────\n    t16_hits, t16_matched = _count_phrase_hits(body_lower, TEMPLATE_16_PHRASES)\n    if t16_hits < TEMPLATE_16_THRESHOLD:\n        issues.append(\n            f\"Payroll handoff (Template 16): matched only {t16_hits}/{TEMPLATE_16_THRESHOLD} \"\n            f\"required phrases (matched: {t16_matched}). \"\n            f\"Expected ≥{TEMPLATE_16_THRESHOLD} of: {TEMPLATE_16_PHRASES[:5]}...\"\n        )\n\n    # ── 4. Visual break – topic headings with blank-line separator ───────\n    bereavement_heading = re.compile(\n        r'(regarding|re:|about|on the topic of|on the matter of|update on|status of)'\n        r'[^\\n]*?(bereavement|leave|absence)',\n        re.IGNORECASE,\n    )\n    pto_heading = re.compile(\n        r'(regarding|re:|about|on the topic of|on the matter of|update on|status of)'\n        r'[^\\n]*?(pto|payroll|pay|famli|supplement|time.off)',\n        re.IGNORECASE,\n    )\n\n    has_bereavement_heading = bool(bereavement_heading.search(body))\n    has_pto_heading = bool(pto_heading.search(body))\n    has_blank_line = bool(re.search(r'\\n\\s*\\n', body))\n\n    if not has_bereavement_heading:\n        issues.append(\n            'No bereavement-related topic heading found '\n            '(expected something like \"Regarding your bereavement leave request:\")'\n        )\n    if not has_pto_heading:\n        issues.append(\n            'No PTO/payroll-related topic heading found '\n            '(expected something like \"Regarding your PTO question:\")'\n        )\n    if not has_blank_line and (has_bereavement_heading or has_pto_heading):\n        issues.append(\"No visual break (blank line) found between topics\")\n\n    # ── Result ───────────────────────────────────────────────────────────\n    if issues:\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": f\"Reply found but missing: {'; '.join(issues)}. Body preview: {body[:500]}\",\n        }\n\n    return {\n        \"pass\": True,\n        \"score\": 1.0,\n        \"feedback\": (\n            \"Found exactly one sent reply to Rosa Gutierrez with: \"\n            f\"Template 31 phrases ({t31_matched}), \"\n            f\"Template 16 phrases ({t16_matched}), \"\n            \"Teresa Vang mentioned by name, \"\n            \"bereavement topic heading present, \"\n            \"PTO/payroll topic heading present, \"\n            \"visual breaks (blank lines) present.\"\n        ),\n    }",
    "criterion_type": "expected_output"
  },
  {
    "id": "e01d14aa-f4d4-4d0f-9a39-cc6346aae71a",
    "sort_order": 1,
    "rubric_text": "In `mailbox.json`, Rosa Gutierrez's email must NOT be forwarded to teresa.vang@ridgelinegear.com, and no manager notification email must be sent to marcus.evans@ridgelinegear.com.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    \n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    \n    with open(mailbox_path) as f:\n        data = json.load(f)\n    \n    emails = data.get(\"emails\", [])\n    \n    sent_to_teresa = []\n    sent_to_marcus = []\n    \n    for e in emails:\n        if e.get(\"folder\", \"\").lower() != \"sent\":\n            continue\n        to_addr = (e.get(\"to_addr\") or \"\").lower()\n        cc_addr = (e.get(\"cc_addr\") or \"\").lower()\n        all_recipients = to_addr + \" \" + cc_addr\n        \n        if \"teresa.vang@ridgelinegear.com\" in all_recipients:\n            sent_to_teresa.append(e)\n        if \"marcus.evans@ridgelinegear.com\" in all_recipients:\n            sent_to_marcus.append(e)\n    \n    issues = []\n    if sent_to_teresa:\n        issues.append(f\"Found {len(sent_to_teresa)} sent email(s) to teresa.vang@ridgelinegear.com (should not be forwarded)\")\n    if sent_to_marcus:\n        issues.append(f\"Found {len(sent_to_marcus)} sent email(s) to marcus.evans@ridgelinegear.com (no manager notification should be sent)\")\n    \n    if issues:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n    \n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Confirmed no emails sent to teresa.vang@ridgelinegear.com or marcus.evans@ridgelinegear.com\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "c00541bc-b57a-49ed-b06d-057405c12b32",
    "sort_order": 2,
    "rubric_text": "In `slack_data.json`, the #benefits-team channel must have exactly 7 posts total, with the most recent post containing the exact content 'BEREAVEMENT ESCALATION: Rosa Gutierrez, Bereavement, 04/20/2026, 04/24/2026'.",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    \n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    \n    with open(slack_path) as f:\n        data = json.load(f)\n    \n    channels = data.get(\"channels\", {})\n    messages = data.get(\"messages\", {})\n    \n    # Find benefits-team channel\n    benefits_channel_id = None\n    for cid, cdata in channels.items():\n        if cdata.get(\"name\", \"\").lower() == \"benefits-team\":\n            benefits_channel_id = cid\n            break\n    \n    if benefits_channel_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"#benefits-team channel not found in slack_data.json\"}\n    \n    channel_messages = messages.get(benefits_channel_id, [])\n    \n    msg_count = len(channel_messages)\n    \n    if msg_count != 7:\n        return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"#benefits-team has {msg_count} messages, expected 7\"}\n    \n    # Find the most recent message (highest timestamp)\n    sorted_msgs = sorted(channel_messages, key=lambda m: float(m.get(\"ts\", 0)), reverse=True)\n    most_recent = sorted_msgs[0] if sorted_msgs else None\n    \n    if most_recent is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #benefits-team\"}\n    \n    recent_text = (most_recent.get(\"text\") or \"\").strip()\n    expected_exact = \"BEREAVEMENT ESCALATION: Rosa Gutierrez, Bereavement, 04/20/2026, 04/24/2026\"\n    \n    # Full credit: text begins with the expected phrase (trailing punctuation/commentary allowed)\n    if recent_text.startswith(expected_exact):\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"#benefits-team has 7 messages, most recent begins with the expected phrase: '{expected_exact}'\"}\n    \n    # Partial credit: correct content but wrong date format\n    recent_lower = recent_text.lower()\n    has_header = \"bereavement escalation\" in recent_lower\n    has_name = \"rosa gutierrez\" in recent_lower\n    has_bereavement = recent_lower.count(\"bereavement\") >= 2  # once in header, once as leave type\n    \n    # Accept any recognizable rendering of April 20, 2026 and April 24, 2026\n    date_start_patterns = [\"04/20/2026\", \"4/20/2026\", \"2026-04-20\", \"april 20, 2026\", \"april 20 2026\", \"04-20-2026\", \"20 april 2026\"]\n    date_end_patterns = [\"04/24/2026\", \"4/24/2026\", \"2026-04-24\", \"april 24, 2026\", \"april 24 2026\", \"04-24-2026\", \"24 april 2026\"]\n    \n    has_start_date = any(p in recent_lower for p in date_start_patterns)\n    has_end_date = any(p in recent_lower for p in date_end_patterns)\n    \n    if has_header and has_name and has_bereavement and has_start_date and has_end_date:\n        return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"#benefits-team has 7 messages, most recent has correct content but wrong date format (SOP §3.1 requires MM/DD/YYYY). Expected exact: '{expected_exact}'. Actual: '{recent_text[:300]}'\"}\n    \n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Most recent #benefits-team post must begin with '{expected_exact}'. Actual text: '{recent_text[:500]}'\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "0475cf23-46b8-416f-86d4-1fa59591a68e",
    "sort_order": 3,
    "rubric_text": "In `slack_data.json`, the #payroll-coordination channel must have exactly 7 posts total, with the most recent post beginning with the exact phrase 'PAYROLL QUESTION: Rosa Gutierrez, ' and ending with a question about PTO to supplement FAMLI payments.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    \n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    \n    with open(slack_path) as f:\n        data = json.load(f)\n    \n    channels = data.get(\"channels\", {})\n    messages = data.get(\"messages\", {})\n    \n    # Find payroll-coordination channel\n    payroll_channel_id = None\n    for cid, cdata in channels.items():\n        if cdata.get(\"name\", \"\").lower() == \"payroll-coordination\":\n            payroll_channel_id = cid\n            break\n    \n    if payroll_channel_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"#payroll-coordination channel not found in slack_data.json\"}\n    \n    channel_messages = messages.get(payroll_channel_id, [])\n    msg_count = len(channel_messages)\n    \n    if msg_count != 7:\n        return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"#payroll-coordination has {msg_count} messages, expected 7\"}\n    \n    # Find the most recent message\n    sorted_msgs = sorted(channel_messages, key=lambda m: float(m.get(\"ts\", 0)), reverse=True)\n    most_recent = sorted_msgs[0] if sorted_msgs else None\n    \n    if most_recent is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #payroll-coordination\"}\n    \n    recent_text = (most_recent.get(\"text\") or \"\").strip()\n    recent_text_lower = recent_text.lower()\n    \n    failures = []\n    \n    # Check prefix\n    if not recent_text_lower.startswith(\"payroll question: rosa gutierrez\"):\n        failures.append(f\"Most recent post does not begin with 'PAYROLL QUESTION: Rosa Gutierrez'. Actual start: '{recent_text[:120]}'\")\n    \n    # Check for PTO content\n    if \"pto\" not in recent_text_lower:\n        failures.append(\"'PTO' not found in most recent post\")\n    \n    # Check for FAMLI content\n    if \"famli\" not in recent_text_lower:\n        failures.append(\"'FAMLI' not found in most recent post\")\n    \n    # Check that the message conveys the concept of using PTO together with / on top of FAMLI.\n    # Accept any reasonable phrasing.\n    combo_terms = [\n        \"supplement\", \"on top of\", \"alongside\", \"in addition to\",\n        \"at the same time\", \"together with\", \"combine\", \"combined\",\n        \"while on\", \"pto+famli\", \"famli+pto\", \"pto + famli\", \"famli + pto\",\n        \"along with\", \"concurrent\", \"stack\", \"use pto\", \"draw pto\",\n        \"add pto\", \"apply pto\", \"use my pto\", \"use her pto\",\n        \"make up the difference\", \"offset\", \"top up\", \"top-up\",\n        \"also receives\", \"also receive\", \"in conjunction\",\n        \"both pto and famli\", \"pto and famli\", \"famli and pto\",\n    ]\n    combo_found = any(term in recent_text_lower for term in combo_terms)\n    if not combo_found:\n        failures.append(\"Post does not convey the concept of using PTO together with FAMLI\")\n    \n    if failures:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Most recent #payroll-coordination post issues: {'; '.join(failures)}. Full text: {recent_text[:500]}\"}\n    \n    return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"#payroll-coordination has 7 messages, most recent begins with 'PAYROLL QUESTION: Rosa Gutierrez' and conveys a question about using PTO with FAMLI. Text: {recent_text[:300]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "7241650b-e2e0-432b-91c7-29f86ea4ddc9",
    "sort_order": 4,
    "rubric_text": "In `slack_data.json`, the #payroll-coordination channel must NOT contain any 'LEAVE PAYROLL NOTIFICATION' post related to Rosa Gutierrez for bereavement.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    \n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    \n    with open(slack_path) as f:\n        data = json.load(f)\n    \n    channels = data.get(\"channels\", {})\n    messages = data.get(\"messages\", {})\n    \n    # Find payroll-coordination channel\n    payroll_channel_id = None\n    for cid, cdata in channels.items():\n        if cdata.get(\"name\", \"\").lower() == \"payroll-coordination\":\n            payroll_channel_id = cid\n            break\n    \n    if payroll_channel_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"#payroll-coordination channel not found in slack_data.json\"}\n    \n    channel_messages = messages.get(payroll_channel_id, [])\n    \n    # Look for any LEAVE PAYROLL NOTIFICATION for Rosa Gutierrez bereavement\n    problematic_msgs = []\n    for msg in channel_messages:\n        text = (msg.get(\"text\") or \"\").lower()\n        if \"leave payroll notification\" in text and \"rosa gutierrez\" in text and \"bereavement\" in text:\n            problematic_msgs.append(msg.get(\"text\", \"\")[:200])\n    \n    if problematic_msgs:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(problematic_msgs)} LEAVE PAYROLL NOTIFICATION message(s) for Rosa Gutierrez bereavement in #payroll-coordination (should not exist): {problematic_msgs[0]}\"}\n    \n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No LEAVE PAYROLL NOTIFICATION for Rosa Gutierrez bereavement found in #payroll-coordination\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "55468c98-2909-42a3-8a51-bf5ec83a2b1a",
    "sort_order": 5,
    "rubric_text": "In `leave_tracker.xlsx` (or equivalent spreadsheet), there must be exactly 47 cases, no new bereavement case for Rosa Gutierrez must be opened, and LC-2026-0089 (Rosa Gutierrez's FMLA + FAMLI case) must remain unchanged with its existing values.",
    "verifier_code": "from pathlib import Path\n\n\ndef verify(workspace_path, external_services_path=None):\n    try:\n        import openpyxl\n    except ImportError:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n    workspace = Path(workspace_path)\n    xlsx_files = list(workspace.glob(\"*.xlsx\")) + list(workspace.glob(\"*.xls\"))\n\n    if not xlsx_files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Excel file found in workspace\"}\n\n    # --- FIX: match the exact filename \"leave_tracker\" rather than\n    # substring \"leave\" or \"tracker\", which incorrectly matched\n    # hfwa_balance_tracker.xlsx before leave_tracker.xlsx. ---\n    leave_tracker = None\n    for f in xlsx_files:\n        if f.stem.lower() == \"leave_tracker\":\n            leave_tracker = f\n            break\n\n    # Fallback: accept any file whose stem starts with \"leave_tracker\"\n    if leave_tracker is None:\n        for f in xlsx_files:\n            if f.stem.lower().startswith(\"leave_tracker\"):\n                leave_tracker = f\n                break\n\n    if leave_tracker is None:\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": (\n                \"leave_tracker.xlsx not found in workspace. \"\n                f\"Files present: {[f.name for f in xlsx_files]}\"\n            ),\n        }\n\n    try:\n        wb = openpyxl.load_workbook(leave_tracker, data_only=True)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {leave_tracker.name}: {e}\"}\n\n    ws = wb.active\n\n    # Collect data rows (skip header and fully-blank rows)\n    data_rows = []\n    header_row = None\n    for row in ws.iter_rows(values_only=True):\n        if all(cell is None for cell in row):\n            continue\n        if header_row is None:\n            header_row = [str(c).strip().lower() if c is not None else \"\" for c in row]\n            continue\n        data_rows.append(row)\n\n    case_count = len(data_rows)\n\n    if case_count != 47:\n        return {\n            \"pass\": False,\n            \"score\": 0.5,\n            \"feedback\": f\"Expected 47 cases in {leave_tracker.name}, found {case_count}\",\n        }\n\n    # Locate LC-2026-0089\n    lc_row = None\n    for row in data_rows:\n        row_str = [str(c).strip() if c is not None else \"\" for c in row]\n        if any(\"lc-2026-0089\" in s.lower() for s in row_str):\n            lc_row = row_str\n            break\n\n    if lc_row is None:\n        return {\"pass\": False, \"score\": 0.5, \"feedback\": \"LC-2026-0089 not found in leave_tracker\"}\n\n    row_text = \" \".join(lc_row).lower()\n\n    checks = [\n        (\"rosa gutierrez\" in row_text, \"Rosa Gutierrez name not in LC-2026-0089 row\"),\n        (\"fmla\" in row_text and \"famli\" in row_text, \"FMLA + FAMLI not in LC-2026-0089 row\"),\n        (\"intermittent\" in row_text, \"Intermittent leave type not in LC-2026-0089 row\"),\n        (\"open\" in row_text, \"Status not 'Open' for LC-2026-0089\"),\n    ]\n\n    failures = [msg for passed, msg in checks if not passed]\n    if failures:\n        return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"LC-2026-0089 row issues: {'; '.join(failures)}\"}\n\n    # Verify no new bereavement case was opened for Rosa Gutierrez\n    bereavement_rosa_cases = []\n    for row in data_rows:\n        row_str = [str(c).strip().lower() if c is not None else \"\" for c in row]\n        row_text_check = \" \".join(row_str)\n        if (\n            \"rosa gutierrez\" in row_text_check\n            and \"bereavement\" in row_text_check\n            and \"lc-2026-0089\" not in row_text_check\n        ):\n            bereavement_rosa_cases.append(row_str)\n\n    if bereavement_rosa_cases:\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": f\"Found {len(bereavement_rosa_cases)} unexpected bereavement case(s) for Rosa Gutierrez\",\n        }\n\n    return {\n        \"pass\": True,\n        \"score\": 1.0,\n        \"feedback\": (\n            f\"{leave_tracker.name} has exactly 47 cases, LC-2026-0089 found with correct values, \"\n            \"no new bereavement case for Rosa Gutierrez\"\n        ),\n    }",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "0ca61599-e6ee-46b7-92cc-77d81d33a2c1",
    "sort_order": 6,
    "rubric_text": "In `mailbox.json`, no actions (reply, forward, mark) must be taken on emails 10 and 12 - no sent emails related to either email.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    \n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    \n    with open(mailbox_path) as f:\n        data = json.load(f)\n    \n    emails = data.get(\"emails\", [])\n    \n    # Find emails 10 and 12 to get their metadata\n    target_emails = {}\n    for e in emails:\n        eid = str(e.get(\"email_id\", \"\"))\n        if eid in (\"10\", \"12\"):\n            target_emails[eid] = e\n    \n    # Collect message_ids, subjects, from addresses for emails 10 and 12\n    target_message_ids = set()\n    target_subjects = set()\n    target_from_addrs = set()\n    target_to_addrs = set()\n    for eid, e in target_emails.items():\n        mid = e.get(\"message_id\", \"\")\n        if mid:\n            target_message_ids.add(mid)\n        subj = (e.get(\"subject\") or \"\").lower()\n        if subj:\n            # Strip common prefixes for matching\n            clean_subj = subj\n            for prefix in [\"re: \", \"fwd: \", \"fw: \"]:\n                while clean_subj.startswith(prefix):\n                    clean_subj = clean_subj[len(prefix):]\n            target_subjects.add(clean_subj)\n        from_addr = (e.get(\"from_addr\") or \"\").lower().strip()\n        if from_addr:\n            target_from_addrs.add(from_addr)\n        to_addr = (e.get(\"to_addr\") or \"\").lower().strip()\n        if to_addr:\n            target_to_addrs.add(to_addr)\n    \n    # Check for any sent emails that are replies/forwards related to emails 10 or 12\n    problematic_sent = []\n    for e in emails:\n        if e.get(\"folder\", \"\").lower() != \"sent\":\n            continue\n        \n        to_addr = (e.get(\"to_addr\") or \"\").lower()\n        body = (e.get(\"body_text\") or \"\").lower()\n        subject = (e.get(\"subject\") or \"\").lower()\n        in_reply_to = e.get(\"in_reply_to\") or \"\"\n        \n        # Check if this sent email is related to email 10 or 12\n        is_related = False\n        reasons = []\n        \n        # Check in_reply_to against target message_ids\n        if in_reply_to:\n            for mid in target_message_ids:\n                if mid in in_reply_to:\n                    is_related = True\n                    reasons.append(f\"in_reply_to matches message_id {mid}\")\n        \n        # Check if sent to original sender of emails 10 or 12\n        # AND subject matches (to avoid false positives)\n        clean_subject = subject\n        for prefix in [\"re: \", \"fwd: \", \"fw: \"]:\n            while clean_subject.startswith(prefix):\n                clean_subject = clean_subject[len(prefix):]\n        \n        if clean_subject in target_subjects:\n            # Subject matches - check if to_addr relates to the original email participants\n            for addr in target_from_addrs:\n                if addr in to_addr:\n                    is_related = True\n                    reasons.append(f\"to_addr contains {addr} and subject matches\")\n            for addr in target_to_addrs:\n                if addr in to_addr:\n                    is_related = True\n                    reasons.append(f\"to_addr contains {addr} and subject matches\")\n        \n        if is_related:\n            problematic_sent.append(f\"To: {e.get('to_addr')}, Subject: {e.get('subject')} (reasons: {'; '.join(reasons)})\")\n    \n    # Also check if emails 10 or 12 have been marked/flagged (if such fields exist)\n    marking_issues = []\n    for eid, e in target_emails.items():\n        # Check common marking fields\n        if e.get(\"is_read\") is True and e.get(\"was_read\") is False:\n            marking_issues.append(f\"Email {eid} was marked as read\")\n        if e.get(\"flagged\") or e.get(\"starred\"):\n            marking_issues.append(f\"Email {eid} was flagged/starred\")\n        if e.get(\"labels_added\") or e.get(\"tags_added\"):\n            marking_issues.append(f\"Email {eid} had labels/tags added\")\n    \n    issues = []\n    if problematic_sent:\n        issues.append(f\"Found {len(problematic_sent)} sent email(s) related to emails 10/12: {'; '.join(problematic_sent)}\")\n    if marking_issues:\n        issues.append(f\"Found marking actions on emails 10/12: {'; '.join(marking_issues)}\")\n    \n    if issues:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Actions were taken on emails 10 and/or 12 (should have no actions): \" + \" | \".join(issues)}\n    \n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails or actions related to emails 10 and 12 found - correct, no actions should be taken on these emails.\"}",
    "criterion_type": "incorrect_behavior"
  }
]