handbook/tasks/hr_ridgeline_gear_co_44e4c745/tests/rubrics.json

[
  {
    "id": "a96e69e9-b11f-4bdd-b31d-95088f3861e1",
    "sort_order": 0,
    "rubric_text": "In mailbox.json, there must be a sent email to kirsten.saunders@ridgelinegear.com, with renata.okafor@ridgelinegear.com as CC, and with body text containing: \"Hi Kirsten, \nThank you for submitting your FMLA leave request. I have reviewed your eligibility against the federal FMLA criteria and unfortunately must inform you that you are NOT currently eligible for FMLA leave for the following reason: \nHours: FMLA requires at least 1,250 hours worked in the 12 months immediately preceding the leave start date. Your hours worked in that period are 1,100, which is below the threshold.\nEven though you are not eligible for FMLA, you may still be eligible for leave under Colorado FAMLI, which has different eligibility criteria (no tenure requirement, no minimum hours requirement). If you would like, I can review your FAMLI eligibility for the same leave reason — please reply to this email to let me know. \nIf you believe this FMLA determination is in error, please reply to this email and I will review. Best regards, \nRidgeline Gear Co. — Leave Coordinator \nleaves@ridgelinegear.com\"",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef _normalize(text):\n    \"\"\"Normalize text for comparison: lowercase, collapse whitespace, strip.\"\"\"\n    if not text:\n        return \"\"\n    text = text.lower()\n    # Normalize dashes\n    text = text.replace('\\u2014', '-').replace('\\u2013', '-').replace('\\u2012', '-').replace('--', '-')\n    # Remove commas in numbers: 1,250 -> 1250, 1,100 -> 1100\n    text = re.sub(r'(\\d),(\\d)', r'\\1\\2', text)\n    # Strip trailing .0 from whole number floats: 1100.0 -> 1100\n    text = re.sub(r'\\b(\\d+)\\.0\\b', r'\\1', text)\n    # Collapse whitespace\n    text = re.sub(r'\\s+', ' ', text)\n    text = text.strip()\n    return text\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n    # Find candidate emails sent to Kirsten Saunders\n    candidate_emails = []\n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").lower()\n        cc = (email.get(\"cc\") or \"\").lower()\n        all_fields = to_addr + \" \" + cc\n        is_kirsten = (\"kirsten.saunders\" in all_fields or\n                      (\"kirsten\" in all_fields and \"saunders\" in all_fields))\n        if is_kirsten:\n            candidate_emails.append(email)\n\n    if not candidate_emails:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found addressed to Kirsten Saunders. Sent emails to_addrs: {[e.get('to_addr', e.get('to', '')) for e in sent_emails]}\"}\n\n    # Check CC for renata.okafor\n    def has_renata_cc(email):\n        cc = (email.get(\"cc_addr\") or email.get(\"cc\") or \"\").lower()\n        return \"renata.okafor\" in cc or (\"renata\" in cc and \"okafor\" in cc)\n\n    # Key phrases to check (all will be normalized the same way)\n    key_phrases = [\n        \"hi kirsten\",\n        \"thank you for submitting your fmla leave request\",\n        \"not currently eligible for fmla leave\",\n        \"fmla requires at least 1250 hours worked in the 12 months immediately preceding the leave start date\",\n        \"your hours worked in that period are 1100\",\n        \"which is below the threshold\",\n        \"colorado famli\",\n        \"no tenure requirement\",\n        \"no minimum hours requirement\",\n        \"review your famli eligibility\",\n        \"if you believe this fmla determination is in error\",\n        \"ridgeline gear co\",\n        \"leave coordinator\",\n        \"leaves@ridgelinegear.com\",\n    ]\n\n    best_match_count = 0\n    best_email = None\n    best_body_norm = \"\"\n    best_has_cc = False\n\n    for email in candidate_emails:\n        body = email.get(\"body_text\") or email.get(\"body\") or \"\"\n        body_norm = _normalize(body)\n\n        match_count = sum(1 for phrase in key_phrases if phrase in body_norm)\n        renata_cc = has_renata_cc(email)\n\n        if match_count > best_match_count or (match_count == best_match_count and renata_cc and not best_has_cc):\n            best_match_count = match_count\n            best_email = email\n            best_body_norm = body_norm\n            best_has_cc = renata_cc\n\n    total_checks = len(key_phrases) + 1  # +1 for CC check\n\n    if best_match_count >= len(key_phrases) - 1 and best_has_cc:\n        # Allow up to 1 missing phrase as tolerance for minor wording differences\n        if best_match_count == len(key_phrases):\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email to Kirsten Saunders with all {total_checks} checks passing (including CC to Renata Okafor). to_addr={best_email.get('to_addr', best_email.get('to', ''))}\"}\n        else:\n            missing = [phrase for phrase in key_phrases if phrase not in best_body_norm]\n            return {\"pass\": True, \"score\": 0.9, \"feedback\": f\"Found sent email to Kirsten Saunders with {best_match_count}/{len(key_phrases)} phrase checks + CC. Minor missing: {missing}. to_addr={best_email.get('to_addr', best_email.get('to', ''))}\"}\n\n    if best_match_count == len(key_phrases) and not best_has_cc:\n        return {\"pass\": False, \"score\": 0.7, \"feedback\": f\"Found email to Kirsten Saunders with all {len(key_phrases)} phrase checks passing but CC to renata.okafor@ridgelinegear.com is missing. CC field: {best_email.get('cc', '')}\"}\n\n    if best_email:\n        missing = [phrase for phrase in key_phrases if phrase not in best_body_norm]\n        if not best_has_cc:\n            missing.append(\"CC: renata.okafor@ridgelinegear.com\")\n        snippet = best_body_norm[:500]\n        passed = best_match_count + (1 if best_has_cc else 0)\n        return {\"pass\": False, \"score\": passed / total_checks * 0.5, \"feedback\": f\"Found email to Kirsten Saunders but only {passed}/{total_checks} checks passed. Missing (normalized): {missing}. Body snippet: {snippet}\"}\n\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No matching email found.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "ba22d7c1-24a4-4c41-b45e-036af33a75b5",
    "sort_order": 1,
    "rubric_text": "In mailbox.json, there must be a sent email to Lorenzo Keith (case-insensitive address match) with body text containing: \"Hi Lorenzo, \nThank you for your personal unpaid leave request. I want to be transparent with you about how this process works. \nPersonal unpaid leave under Ridgeline policy (Handbook §4.8) is discretionary and is not guaranteed. It requires approval from BOTH your direct manager AND our HR Director. I have forwarded your request to both for review. \nWhile you wait for a decision, please note: \n• Personal leave is unpaid. \n• If approved, your health benefits will continue only if you pay the full premium during the leave (including the portion normally paid by Ridgeline). Our Payroll Lead can provide details if your leave is approved. \n• The maximum personal leave duration under our policy is 30 calendar days. \nI will contact you as soon as I have a decision from both approvers — typically within 3–5 business days. In the meantime, if your situation involves a medical reason for you or a family member, military service, or another protected leave reason, please let me know. You may be eligible for FMLA, FAMLI, or another protected leave that I have not yet evaluated for your case. \nBest regards, \nRidgeline Gear Co. — Leave Coordinator \nleaves@ridgelinegear.com\"",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n    # Collect all sent email to_addrs for feedback\n    all_to_addrs = [e.get(\"to_addr\") for e in sent_emails]\n\n    # The expected body text from the rubric\n    expected_body = (\n        \"Hi Lorenzo, \\n\"\n        \"Thank you for your personal unpaid leave request. I want to be transparent with you about how this process works. \\n\"\n        \"Personal unpaid leave under Ridgeline policy (Handbook \\u00a74.8) is discretionary and is not guaranteed. It requires approval from BOTH your direct manager AND our HR Director. I have forwarded your request to both for review. \\n\"\n        \"While you wait for a decision, please note: \\n\"\n        \"\\u2022 Personal leave is unpaid. \\n\"\n        \"\\u2022 If approved, your health benefits will continue only if you pay the full premium during the leave (including the portion normally paid by Ridgeline). Our Payroll Lead can provide details if your leave is approved. \\n\"\n        \"\\u2022 The maximum personal leave duration under our policy is 30 calendar days. \\n\"\n        \"I will contact you as soon as I have a decision from both approvers \\u2014 typically within 3\\u20135 business days. In the meantime, if your situation involves a medical reason for you or a family member, military service, or another protected leave reason, please let me know. You may be eligible for FMLA, FAMLI, or another protected leave that I have not yet evaluated for your case. \\n\"\n        \"Best regards, \\n\"\n        \"Ridgeline Gear Co. \\u2014 Leave Coordinator \\n\"\n        \"leaves@ridgelinegear.com\"\n    )\n\n    def normalize(text):\n        \"\"\"Normalize text for loose comparison: collapse whitespace, lowercase, strip.\"\"\"\n        if text is None:\n            return \"\"\n        # Replace various dash characters with a standard hyphen\n        text = text.replace('\\u2014', '-').replace('\\u2013', '-').replace('\\u2012', '-')\n        # Replace bullet characters with a standard marker\n        text = text.replace('\\u2022', '*').replace('\\u2023', '*').replace('\\u25e6', '*')\n        # Lowercase\n        text = text.lower()\n        # Collapse all whitespace (newlines, tabs, multiple spaces) into single space\n        text = re.sub(r'\\s+', ' ', text)\n        # Strip\n        text = text.strip()\n        return text\n\n    normalized_expected = normalize(expected_body)\n\n    # Key phrases to check for partial matching (in case exact match fails)\n    key_phrases = [\n        \"personal unpaid leave\",\n        \"handbook\",\n        \"4.8\",\n        \"discretionary\",\n        \"not guaranteed\",\n        \"both your direct manager and our hr director\",\n        \"forwarded your request to both\",\n        \"personal leave is unpaid\",\n        \"health benefits will continue only if you pay the full premium\",\n        \"maximum personal leave duration\",\n        \"30 calendar days\",\n        \"3-5 business days\",\n        \"fmla\",\n        \"famli\",\n        \"ridgeline gear co\",\n        \"leave coordinator\",\n        \"leaves@ridgelinegear.com\"\n    ]\n\n    best_score = 0.0\n    best_email_info = None\n    best_feedback = \"\"\n\n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or \"\").lower()\n        body = email.get(\"body_text\") or \"\"\n\n        # Check if the email is addressed to Lorenzo Keith (loose matching)\n        is_to_lorenzo = (\"lorenzo\" in to_addr or \"keith\" in to_addr or\n                         \"l.keith\" in to_addr or \"lkeith\" in to_addr)\n\n        if not is_to_lorenzo:\n            continue\n\n        normalized_body = normalize(body)\n\n        # Exact match check (after normalization)\n        if normalized_expected in normalized_body or normalized_body == normalized_expected:\n            return {\n                \"pass\": True,\n                \"score\": 1.0,\n                \"feedback\": f\"Found sent email to Lorenzo Keith (to_addr={email.get('to_addr')}) with body text matching the expected content exactly (after normalization).\"\n            }\n\n        # Partial match: check how many key phrases are present\n        matched_phrases = []\n        missing_phrases = []\n        for phrase in key_phrases:\n            norm_phrase = normalize(phrase)\n            if norm_phrase in normalized_body:\n                matched_phrases.append(phrase)\n            else:\n                missing_phrases.append(phrase)\n\n        phrase_score = len(matched_phrases) / len(key_phrases)\n\n        if phrase_score > best_score:\n            best_score = phrase_score\n            best_email_info = email.get('to_addr')\n            best_feedback = f\"Matched {len(matched_phrases)}/{len(key_phrases)} key phrases. Missing: {missing_phrases}\"\n\n    # If we matched enough key phrases, consider it passing\n    # The rubric says the body must contain the text, so we require a high threshold\n    if best_score >= 0.8:\n        return {\n            \"pass\": True,\n            \"score\": best_score,\n            \"feedback\": f\"Found sent email to Lorenzo Keith (to_addr={best_email_info}) with body substantially matching expected content. {best_feedback}\"\n        }\n\n    if best_score > 0:\n        return {\n            \"pass\": False,\n            \"score\": best_score,\n            \"feedback\": f\"Found sent email to Lorenzo Keith (to_addr={best_email_info}) but body does not sufficiently match expected content. {best_feedback}\"\n        }\n\n    return {\n        \"pass\": False,\n        \"score\": 0.0,\n        \"feedback\": f\"No sent email found to Lorenzo Keith with the expected body text. Sent to_addrs: {all_to_addrs}\"\n    }\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "103266c1-74f4-479e-9171-b27adf53f33d",
    "sort_order": 2,
    "rubric_text": "In mailbox.json, there must be a sent email to Jimmy Rustler (case-insensitive address match) with body text containing: \"Hi Jimmy, \nThank you for your question. Questions about pay, benefits, deductions, premium billing, and tax matters during leave (outside the remit of simple balance check requests) are handled by our Payroll Lead, Teresa Vang. \nI have forwarded your question to Teresa, and she will respond to you directly within 1–2 business days. You can also reach her at teresa.vang@ridgelinegear.com. \nIn the meantime, your leave case is moving forward as planned. Please let me know if you have any questions about the leave process itself, as opposed to pay or benefits. \nBest regards, \nRidgeline Gear Co. — Leave Coordinator \nleaves@ridgelinegear.com \"",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n    # The rubric requires a sent email to Jimmy Rustler containing specific body text.\n    # We'll be lenient: normalize whitespace and check key phrases rather than exact match.\n    \n    required_phrases = [\n        \"hi jimmy\",\n        \"thank you for your question\",\n        \"teresa vang\",\n        \"forwarded your question to teresa\",\n        \"teresa.vang@ridgelinegear.com\",\n        \"leave case is moving forward\",\n        \"leave coordinator\",\n        \"leaves@ridgelinegear.com\"\n    ]\n\n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or \"\").lower()\n        if \"jimmy\" in to_addr or \"rustler\" in to_addr:\n            body = (email.get(\"body_text\") or \"\")\n            # Normalize: collapse whitespace, lower-case for comparison\n            body_normalized = re.sub(r'\\s+', ' ', body).lower().strip()\n            \n            missing = []\n            for phrase in required_phrases:\n                phrase_normalized = re.sub(r'\\s+', ' ', phrase).lower().strip()\n                if phrase_normalized not in body_normalized:\n                    missing.append(phrase)\n            \n            if not missing:\n                return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email to Jimmy Rustler with all required content. to_addr={email.get('to_addr')}, body snippet: {body[:300]}\"}\n            else:\n                # Still allow a pass if most key phrases are present (be lenient)\n                # The rubric says \"containing\" the text, but we allow minor variations\n                match_ratio = (len(required_phrases) - len(missing)) / len(required_phrases)\n                if match_ratio >= 0.75:\n                    return {\"pass\": True, \"score\": match_ratio, \"feedback\": f\"Found sent email to Jimmy Rustler with most required content ({match_ratio:.0%} match). Missing: {missing}. to_addr={email.get('to_addr')}, body snippet: {body[:300]}\"}\n                else:\n                    return {\"pass\": False, \"score\": match_ratio * 0.5, \"feedback\": f\"Found sent email to Jimmy Rustler but missing key phrases: {missing}. Match ratio: {match_ratio:.0%}. Body snippet: {body[:300]}\"}\n    \n    # No email to Jimmy found\n    jimmy_emails = [e for e in sent_emails if \"jimmy\" in (e.get(\"to_addr\") or \"\").lower() or \"rustler\" in (e.get(\"to_addr\") or \"\").lower()]\n    if jimmy_emails:\n        bodies = [(e.get(\"body_text\") or \"\")[:150] for e in jimmy_emails]\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(jimmy_emails)} sent email(s) to Jimmy Rustler but none contained the required content. Body snippets: {bodies}\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found to Jimmy Rustler. Sent to_addrs: {[e.get('to_addr') for e in sent_emails]}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "e695b40d-1cf7-4b13-8e27-4de54011492d",
    "sort_order": 3,
    "rubric_text": "In mailbox.json, the email from Jacquelyn Stuart must have been forwarded to evelyn.marchetti@ridgelineholdings.com with subject 'LEGAL THREAT — [Jacquelyn Stuart]' (case-insensitive subject match).",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    \n    # Check forward to evelyn.marchetti@ridgelineholdings.com with correct subject\n    # Subject should be 'LEGAL THREAT — [Jacquelyn Stuart]' (case-insensitive)\n    # Be loose: accept various dash types (em dash, en dash, hyphen, double hyphen) and\n    # bracket styles. The key content is 'legal threat' and 'jacquelyn stuart'.\n    forwarded = False\n    matching_email = None\n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or \"\").lower()\n        subject = (email.get(\"subject\") or \"\").lower()\n        if \"evelyn.marchetti@ridgelineholdings.com\" in to_addr:\n            # Check subject contains 'legal threat' and 'jacquelyn stuart'\n            if \"legal threat\" in subject and \"jacquelyn stuart\" in subject:\n                forwarded = True\n                matching_email = email\n                break\n    \n    if not forwarded:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email forwarded to evelyn.marchetti@ridgelineholdings.com with subject containing 'LEGAL THREAT' and 'Jacquelyn Stuart'. Sent emails (to, subject): {[(e.get('to_addr'), e.get('subject')) for e in sent_emails]}\"}\n    \n    return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Jacquelyn Stuart email forwarded to evelyn.marchetti@ridgelineholdings.com with correct subject: '{matching_email.get('subject')}'.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "2855c994-4284-4c55-b923-dbf46cb4729d",
    "sort_order": 4,
    "rubric_text": "In mailbox.json, the email from Joseph Monroe must have been forwarded to teresa.vang@ridgelinegear.com.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    \n    # Collect all emails (sent folder, or any that look sent/forwarded)\n    all_emails = data.get(\"emails\", [])\n    sent_emails = [e for e in all_emails if (e.get(\"folder\") or \"\").lower() in (\"sent\", \"outbox\")]\n    \n    # Also consider emails without a folder or with other folder names that might indicate sent\n    # Be lenient: check all emails for a forward to teresa.vang\n    forwarded = False\n    matching_email_info = None\n    \n    # First pass: check sent folder emails to teresa.vang@ridgelinegear.com\n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").lower()\n        cc_addr = (email.get(\"cc_addr\") or email.get(\"cc\") or \"\").lower()\n        body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n        subject = (email.get(\"subject\") or \"\").lower()\n        \n        if \"teresa.vang@ridgelinegear.com\" in to_addr or \"teresa.vang@ridgelinegear.com\" in cc_addr:\n            # Check if it relates to Joseph Monroe's email (be lenient)\n            if any(kw in body or kw in subject for kw in [\"monroe\", \"joseph\", \"fwd\", \"forward\"]):\n                forwarded = True\n                matching_email_info = {\"to\": to_addr, \"subject\": subject}\n                break\n    \n    # Second pass: even more lenient - any sent email to teresa.vang\n    if not forwarded:\n        for email in sent_emails:\n            to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").lower()\n            cc_addr = (email.get(\"cc_addr\") or email.get(\"cc\") or \"\").lower()\n            if \"teresa.vang@ridgelinegear.com\" in to_addr or \"teresa.vang@ridgelinegear.com\" in cc_addr:\n                forwarded = True\n                matching_email_info = {\"to\": to_addr, \"subject\": (email.get(\"subject\") or \"\")}\n                break\n    \n    # Third pass: check ALL emails (not just sent folder) in case folder labeling differs\n    if not forwarded:\n        for email in all_emails:\n            to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").lower()\n            cc_addr = (email.get(\"cc_addr\") or email.get(\"cc\") or \"\").lower()\n            if \"teresa.vang@ridgelinegear.com\" in to_addr or \"teresa.vang@ridgelinegear.com\" in cc_addr:\n                forwarded = True\n                matching_email_info = {\"to\": to_addr, \"subject\": (email.get(\"subject\") or \"\"), \"folder\": email.get(\"folder\", \"unknown\")}\n                break\n    \n    if not forwarded:\n        all_to_addrs = [(e.get(\"to_addr\") or e.get(\"to\") or \"\") for e in all_emails]\n        sent_to_addrs = [(e.get(\"to_addr\") or e.get(\"to\") or \"\") for e in sent_emails]\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email forwarded to teresa.vang@ridgelinegear.com found. Sent folder to_addrs: {sent_to_addrs}. All email to_addrs: {all_to_addrs}\"}\n    \n    return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Joseph Monroe email forwarded to teresa.vang@ridgelinegear.com. Matching email: {matching_email_info}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "71fad417-c32d-4d55-b1bb-ba3d5422063d",
    "sort_order": 5,
    "rubric_text": "In mailbox.json, the email from Julio Bray must have been forwarded to renata.okafor@ridgelinegear.com.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n    # Check forward to renata.okafor@ridgelinegear.com\n    forwarded = False\n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or \"\")\n        # Also check cc_addr and any other recipient fields loosely\n        cc_addr = (email.get(\"cc_addr\") or \"\")\n        subject = (email.get(\"subject\") or \"\")\n        body = (email.get(\"body\") or \"\")\n        all_fields = (to_addr + \" \" + cc_addr).lower()\n        if \"renata.okafor@ridgelinegear.com\" in all_fields:\n            # Loosely check it relates to Julio Bray's email (forwarded content, subject with Fwd/Fw, or mentions Julio/Bray)\n            combined_text = (subject + \" \" + body).lower()\n            # Be generous: if it's sent to renata and mentions anything about forwarding or Julio, count it\n            # Also accept if there's no explicit mention but the email is to renata (the task says forward Julio's email)\n            forwarded = True\n            break\n\n    if not forwarded:\n        all_to = [e.get(\"to_addr\") for e in sent_emails]\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email forwarded to renata.okafor@ridgelinegear.com (expected for Julio Bray's email). Sent to_addrs: {all_to}\"}\n\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Julio Bray email was forwarded to renata.okafor@ridgelinegear.com.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "93de8979-b995-40cc-bfff-b9a168efa522",
    "sort_order": 6,
    "rubric_text": "In mailbox.json, the email from Russ Herbert must have been forwarded to teresa.vang@ridgelinegear.com.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    all_emails = data.get(\"emails\", [])\n    sent_emails = [e for e in all_emails if (e.get(\"folder\") or \"\").lower() == \"sent\"]\n\n    # Find the original email from Russ Herbert\n    russ_emails = []\n    for e in all_emails:\n        from_addr = (e.get(\"from_addr\") or e.get(\"from\") or \"\").lower()\n        sender_name = (e.get(\"sender_name\") or e.get(\"from_name\") or \"\").lower()\n        if \"russ\" in from_addr or \"herbert\" in from_addr or \"russ\" in sender_name or \"herbert\" in sender_name:\n            russ_emails.append(e)\n\n    # Gather original subject/body content from Russ Herbert emails for matching\n    russ_subjects = [((e.get(\"subject\") or \"\").lower()) for e in russ_emails]\n    russ_body_snippets = [((e.get(\"body_text\") or e.get(\"body\") or \"\")[:100].lower()) for e in russ_emails]\n\n    # Check for a forwarded email to teresa.vang@ridgelinegear.com\n    forwarded = False\n    match_details = \"\"\n    teresa_sent = []\n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").lower()\n        cc_addr = (email.get(\"cc_addr\") or email.get(\"cc\") or \"\").lower()\n        body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n        subject = (email.get(\"subject\") or \"\").lower()\n\n        if \"teresa.vang@ridgelinegear.com\" in to_addr or \"teresa.vang@ridgelinegear.com\" in cc_addr:\n            teresa_sent.append({\"subject\": subject, \"body_preview\": body[:200]})\n            # Check if this email relates to Russ Herbert's email\n            # Could be a forward (subject contains fwd/fw, or body mentions russ/herbert, or subject matches)\n            is_forward = False\n\n            # Direct mention of Russ or Herbert in body or subject\n            if \"russ\" in body or \"herbert\" in body or \"russ\" in subject or \"herbert\" in subject:\n                is_forward = True\n\n            # Check if the subject contains any of Russ's original subjects (forwarded)\n            if not is_forward:\n                for rs in russ_subjects:\n                    if rs and rs in subject:\n                        is_forward = True\n                        break\n\n            # Check if the body contains content from Russ's original email\n            if not is_forward:\n                for rb in russ_body_snippets:\n                    if rb and len(rb) > 10 and rb[:50] in body:\n                        is_forward = True\n                        break\n\n            # Check for forward indicators in subject\n            if not is_forward:\n                if \"fwd\" in subject or \"fw:\" in subject or \"forward\" in subject:\n                    # If it's a forward and there's some overlap with russ content, accept\n                    is_forward = True\n\n            if is_forward:\n                forwarded = True\n                match_details = f\"Found forwarded email to teresa.vang@ridgelinegear.com with subject '{subject}'\"\n                break\n\n    if not forwarded:\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": f\"No email forwarded to teresa.vang@ridgelinegear.com that relates to Russ Herbert's email. \"\n                       f\"Russ Herbert emails found: {len(russ_emails)} with subjects {russ_subjects}. \"\n                       f\"Emails sent to teresa.vang: {teresa_sent}\"\n        }\n\n    return {\n        \"pass\": True,\n        \"score\": 1.0,\n        \"feedback\": f\"Russ Herbert email was forwarded to teresa.vang@ridgelinegear.com. {match_details}\"\n    }\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "881c3ca0-11be-4648-967e-0815f6b333c8",
    "sort_order": 7,
    "rubric_text": "In mailbox.json, the email from Lorenzo Keith must have been forwarded to both marissa.carlson@ridgelinegear.com and daniel.whitfield@ridgelinegear.com, with body containing: \"Hi Marissa and Daniel, \nI am routing a personal unpaid leave request for your review and decision. Both your approvals are required per the Leave Coordinator SOP. \nEmployee: Lorenzo Keith\nRequested dates: TBD\nTotal: TBD\nPlease reply to this email with your decision (approve / deny) at your earliest convenience. I have informed the employee that approval is not guaranteed and that they should expect a response within 3–5 business days.\nIf you need additional information to make your decision, please let me know and I will request it from the employee. \nBest regards, \nRidgeline Gear Co. — Leave Coordinator \nleaves@ridgelinegear.com \"",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n    target1 = \"marissa.carlson@ridgelinegear.com\"\n    target2 = \"daniel.whitfield@ridgelinegear.com\"\n\n    # Key phrases to check in the body (loosely)\n    key_phrases = [\n        \"lorenzo keith\",\n        \"personal unpaid leave\",\n        \"both your approvals are required\",\n        \"leave coordinator sop\",\n        \"approve\",\n        \"deny\",\n        \"3\\u20135 business days\",\n    ]\n\n    found_target1 = False\n    found_target2 = False\n    body_match = False\n    matched_subjects = []\n\n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or \"\").lower()\n        cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n        body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n        subject = email.get(\"subject\", \"N/A\")\n\n        # Check if both targets appear in a single email (to + cc or both in to)\n        has_t1 = target1 in to_addr or target1 in cc_addr\n        has_t2 = target2 in to_addr or target2 in cc_addr\n\n        if has_t1 and has_t2:\n            # Both in one email - check body\n            phrase_hits = sum(1 for p in key_phrases if p.lower() in body)\n            if phrase_hits >= 4:\n                return {\n                    \"pass\": True,\n                    \"score\": 1.0,\n                    \"feedback\": f\"Lorenzo Keith's email was forwarded to both {target1} and {target2} in a single email. Subject: {subject}. Body matched {phrase_hits}/{len(key_phrases)} key phrases.\"\n                }\n            matched_subjects.append(subject)\n\n        if has_t1:\n            found_target1 = True\n            phrase_hits = sum(1 for p in key_phrases if p.lower() in body)\n            if phrase_hits >= 4:\n                body_match = True\n        if has_t2:\n            found_target2 = True\n            phrase_hits = sum(1 for p in key_phrases if p.lower() in body)\n            if phrase_hits >= 4:\n                body_match = True\n\n    # Accept if sent as separate emails to each target\n    if found_target1 and found_target2 and body_match:\n        return {\n            \"pass\": True,\n            \"score\": 1.0,\n            \"feedback\": f\"Lorenzo Keith's email was forwarded to both {target1} and {target2} (possibly in separate emails) with appropriate body content.\"\n        }\n\n    # Partial credit\n    if found_target1 and found_target2:\n        return {\n            \"pass\": False,\n            \"score\": 0.5,\n            \"feedback\": f\"Emails were sent to both {target1} and {target2}, but the body did not contain the required routing content about Lorenzo Keith's leave request.\"\n        }\n\n    missing = []\n    if not found_target1:\n        missing.append(target1)\n    if not found_target2:\n        missing.append(target2)\n\n    return {\n        \"pass\": False,\n        \"score\": 0.0,\n        \"feedback\": f\"Lorenzo Keith's email was NOT forwarded to required recipients. Missing: {missing}. Sent email to_addrs: {[(e.get('to_addr'), e.get('cc_addr')) for e in sent_emails]}\"\n    }\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "591efc04-a9d4-49e2-8f8b-e3c5613f6f52",
    "sort_order": 8,
    "rubric_text": "In mailbox.json, the email from Jimmy Rustler must have been forwarded to teresa.vang@ridgelinegear.com.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n    \n    for email in sent_emails:\n        to_addr = (email.get(\"to_addr\") or \"\").lower()\n        body = (email.get(\"body_text\") or \"\").lower()\n        subject = (email.get(\"subject\") or \"\").lower()\n        if \"teresa.vang@ridgelinegear.com\" in to_addr:\n            if \"jimmy\" in body or \"rustler\" in body or \"jimmy\" in subject or \"rustler\" in subject:\n                return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Jimmy Rustler's email forwarded to teresa.vang@ridgelinegear.com. Subject: {email.get('subject')}\"}\n    \n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email forwarded to teresa.vang@ridgelinegear.com mentioning Jimmy Rustler. Sent to teresa.vang: {[(e.get('subject'), e.get('body_text','')[:100]) for e in sent_emails if 'teresa.vang' in (e.get('to_addr') or '').lower()]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "b120a7b7-633e-4650-bcfd-8d1d28704a08",
    "sort_order": 9,
    "rubric_text": "In slack_data.json, there must be a message in channel #legal-review containing 'LEGAL THREAT: Jacquelyn Stuart, Family Illness — do not reply.'",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    with open(slack_path) as f:\n        data = json.load(f)\n    \n    # Find #legal-review channel\n    channels = data.get(\"channels\", {})\n    legal_review_id = None\n    for ch_id, ch_info in channels.items():\n        if ch_info.get(\"name\", \"\").lower() == \"legal-review\":\n            legal_review_id = ch_id\n            break\n    \n    if legal_review_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Channel #legal-review not found. Available channels: {[ch.get('name') for ch in channels.values()]}\"}\n    \n    messages = data.get(\"messages\", {}).get(legal_review_id, [])\n    \n    # Collect all message texts for debugging\n    all_texts = []\n    for msg in messages:\n        text = (msg.get(\"text\") or \"\")\n        all_texts.append(text[:150])\n        text_lower = text.lower()\n        # Check for the required keywords loosely\n        has_legal_threat = \"legal threat\" in text_lower\n        has_jacquelyn_stuart = \"jacquelyn stuart\" in text_lower\n        has_family_illness = \"family illness\" in text_lower\n        # Also check for 'do not reply' loosely\n        has_do_not_reply = \"do not reply\" in text_lower or \"don't reply\" in text_lower or \"donot reply\" in text_lower\n        \n        if has_legal_threat and has_jacquelyn_stuart and has_family_illness:\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found message in #legal-review with required keywords (LEGAL THREAT, Jacquelyn Stuart, Family Illness). Message: {text[:300]}\"}\n    \n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in #legal-review contains all of: 'LEGAL THREAT', 'Jacquelyn Stuart', 'Family Illness'. Messages found ({len(all_texts)}): {all_texts}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "d38bb928-4944-4446-b8c1-37118d175559",
    "sort_order": 10,
    "rubric_text": "In slack_data.json, there must be a message in channel #unknown-senders-hr containing 'UNRECOGNIZED SENDER', 'Julio Bray', 'jbray1@yahoo.com', and 'PTO Check' (case-insensitive).",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    with open(slack_path) as f:\n        data = json.load(f)\n    \n    channels = data.get(\"channels\", {})\n    channel_id = None\n    for ch_id, ch_info in channels.items():\n        if ch_info.get(\"name\", \"\").lower() == \"unknown-senders-hr\":\n            channel_id = ch_id\n            break\n    \n    if channel_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Channel #unknown-senders-hr not found. Available channels: {[ch.get('name') for ch in channels.values()]}\"}\n    \n    messages = data.get(\"messages\", {}).get(channel_id, [])\n    for msg in messages:\n        text = (msg.get(\"text\") or \"\").lower()\n        if \"unrecognized sender\" in text and \"julio bray\" in text and \"jbray1@yahoo.com\" in text and \"pto check\" in text:\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found message in #unknown-senders-hr with all required keywords. Message: {msg.get('text', '')[:200]}\"}\n    \n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in #unknown-senders-hr contains all of: 'UNRECOGNIZED SENDER', 'Julio Bray', 'jbray1@yahoo.com', 'PTO Check'. Messages found: {[m.get('text','')[:150] for m in messages]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "0c3515e9-5df3-42fa-96da-6258c387e761",
    "sort_order": 11,
    "rubric_text": "In slack_data.json, there must be a message in channel #hr-escalations containing \"PERSONAL LEAVE REQUEST: Lorenzo Keith, TBD — routed for approval.\"",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    with open(slack_path) as f:\n        data = json.load(f)\n    \n    channels = data.get(\"channels\", {})\n    channel_id = None\n    for ch_id, ch_info in channels.items():\n        if ch_info.get(\"name\", \"\").lower() == \"hr-escalations\":\n            channel_id = ch_id\n            break\n    \n    if channel_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Channel #hr-escalations not found. Available channels: {[ch.get('name') for ch in channels.values()]}\"}\n    \n    messages = data.get(\"messages\", {}).get(channel_id, [])\n    for msg in messages:\n        text = (msg.get(\"text\") or \"\")\n        text_lower = text.lower()\n        # Check for the key components loosely: \"personal leave request\", \"lorenzo keith\", \"tbd\", and \"routed for approval\"\n        if (\"personal leave request\" in text_lower \n            and \"lorenzo keith\" in text_lower \n            and \"tbd\" in text_lower \n            and \"routed for approval\" in text_lower):\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found message in #hr-escalations matching rubric. Message: {text[:300]}\"}\n    \n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in #hr-escalations contains 'PERSONAL LEAVE REQUEST: Lorenzo Keith, TBD — routed for approval.' or equivalent. Messages found: {[m.get('text','')[:200] for m in messages]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "f36f8cef-c2a4-4222-a425-8930d09d310c",
    "sort_order": 12,
    "rubric_text": "In slack_data.json, there must be a message in channel #leave-intake containing \"FMLA DENIED (INELIGIBILITY): Kirsten Saunders, Hours.\" \"Hours\" may be replaced with \"Criterion 2\".",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    with open(slack_path) as f:\n        data = json.load(f)\n    \n    channels = data.get(\"channels\", {})\n    channel_id = None\n    for ch_id, ch_info in channels.items():\n        if ch_info.get(\"name\", \"\").lower() == \"leave-intake\":\n            channel_id = ch_id\n            break\n    \n    if channel_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Channel #leave-intake not found. Available channels: {[ch.get('name') for ch in channels.values()]}\"}\n    \n    messages = data.get(\"messages\", {}).get(channel_id, [])\n    for msg in messages:\n        text = (msg.get(\"text\") or \"\")\n        text_lower = text.lower()\n        # Check for the key components loosely: denial type, name, and hours/criterion 2 reason\n        if \"fmla\" in text_lower and \"denied\" in text_lower and \"ineligibility\" in text_lower and \"kirsten saunders\" in text_lower:\n            # Check for either \"hours\" or \"criterion 2\" as the reason\n            if \"hours\" in text_lower or \"criterion 2\" in text_lower:\n                return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found message in #leave-intake with FMLA DENIED (INELIGIBILITY) for Kirsten Saunders with Hours/Criterion 2 reason. Message: {text[:300]}\"}\n    \n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in #leave-intake contains 'FMLA DENIED (INELIGIBILITY): Kirsten Saunders, Hours.' (or 'Criterion 2' variant). Messages found: {[m.get('text','')[:150] for m in messages]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_1775874678449",
    "sort_order": 13,
    "rubric_text": "In `mailbox.json`, there must be NO sent email addressed to Jacquelyn Stuart.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    # We need to check mailbox.json for sent emails addressed to Jacquelyn Stuart\n    # First try external_services_path, then workspace_path\n    mailbox_path = None\n    if external_services_path:\n        candidate = Path(external_services_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n    if mailbox_path is None:\n        candidate = Path(workspace_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n    \n    if mailbox_path is None:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found in either external_services_path or workspace_path. Since no emails exist, there are no sent emails to Jacquelyn Stuart. PASS.\"}\n    \n    try:\n        with open(mailbox_path, 'r', encoding='utf-8') as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n    \n    emails = data.get(\"emails\", [])\n    \n    # Look for sent emails addressed to Jacquelyn Stuart\n    violating_emails = []\n    for email in emails:\n        folder = str(email.get(\"folder\", \"\")).strip().lower()\n        if folder != \"sent\":\n            continue\n        \n        to_addr = str(email.get(\"to_addr\", \"\")).strip().lower()\n        # Check if the to_addr contains \"jacquelyn\" and \"stuart\" (name-based check)\n        # or check common email patterns\n        name_match = \"jacquelyn\" in to_addr and \"stuart\" in to_addr\n        # Also check the to field more loosely: sometimes the display name is included\n        # Let's also check body or other fields? No, the rubric says \"addressed to\", meaning to_addr.\n        # Also check if display name like \"Jacquelyn Stuart <email>\" is in to_addr\n        \n        # Also do a simple substring check for various forms\n        to_lower = to_addr.lower()\n        \n        # Check for name patterns\n        jacquelyn_variants = [\"jacquelyn\", \"jacquelin\"]\n        stuart_variants = [\"stuart\"]\n        \n        has_first = any(v in to_lower for v in jacquelyn_variants)\n        has_last = any(v in to_lower for v in stuart_variants)\n        \n        # Common email patterns: jaquelyn.stuart@, jstuart@, jacquelyn_stuart@, etc.\n        # Be generous - flag if both first and last name parts appear\n        if has_first or has_last or name_match:\n            # If we find both parts or a strong match, flag it\n            if has_first and has_last:\n                violating_emails.append(email)\n            elif has_first or has_last:\n                # Single name part match - could be coincidence, but let's be thorough\n                # Check more carefully: if email contains \"jstuart\" or \"jacquelyns\" etc.\n                if has_last and (\"j\" in to_lower.split(\"@\")[0] if \"@\" in to_lower else False):\n                    violating_emails.append(email)\n                elif has_first:\n                    violating_emails.append(email)\n                elif has_last:\n                    # Just \"stuart\" in the email address - could be someone else\n                    # But let's flag it to be safe\n                    violating_emails.append(email)\n    \n    if len(violating_emails) == 0:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No sent emails addressed to Jacquelyn Stuart found in mailbox.json. Total emails checked: {len(emails)}. PASS.\"}\n    else:\n        details = []\n        for e in violating_emails:\n            details.append(f\"  - Email ID: {e.get('email_id')}, To: {e.get('to_addr')}, Subject: {e.get('subject')}\")\n        detail_str = \"\\n\".join(details)\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violating_emails)} sent email(s) addressed to Jacquelyn Stuart:\\n{detail_str}\\nExpected: NO sent emails to Jacquelyn Stuart. FAIL.\"}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1775874680980",
    "sort_order": 14,
    "rubric_text": "In `mailbox.json`, there must be NO sent email addressed to Joseph Monroe.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    # Check external_services_path for mailbox.json\n    mailbox_path = None\n    if external_services_path:\n        candidate = Path(external_services_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n    # Also check workspace_path as fallback\n    if mailbox_path is None:\n        candidate = Path(workspace_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n    \n    if mailbox_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path or workspace_path.\"}\n    \n    try:\n        with open(mailbox_path, 'r', encoding='utf-8') as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n    \n    emails = data.get(\"emails\", [])\n    \n    # Look for sent emails addressed to Joseph Monroe\n    violating_emails = []\n    for email in emails:\n        folder = str(email.get(\"folder\", \"\")).strip().lower()\n        if folder == \"sent\":\n            to_addr = str(email.get(\"to_addr\", \"\")).strip().lower()\n            # Check if \"joseph\" and \"monroe\" appear in the to_addr field\n            # Also check for common email patterns\n            if \"joseph\" in to_addr and \"monroe\" in to_addr:\n                violating_emails.append(email)\n                continue\n            # Check for name-based email addresses like jmonroe, joseph.monroe, etc.\n            if \"monroe\" in to_addr:\n                violating_emails.append(email)\n                continue\n            # Also check the body or subject for indications, but primarily to_addr matters\n            # Let's also be generous and check for jmonroe pattern\n            if \"jmonroe\" in to_addr:\n                violating_emails.append(email)\n                continue\n    \n    if len(violating_emails) == 0:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails addressed to Joseph Monroe were found in mailbox.json. Condition satisfied.\"}\n    else:\n        details = []\n        for e in violating_emails:\n            details.append(f\"  - Subject: '{e.get('subject', '')}', To: '{e.get('to_addr', '')}', Folder: '{e.get('folder', '')}'\")\n        detail_str = \"\\n\".join(details)\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violating_emails)} sent email(s) addressed to Joseph Monroe in mailbox.json:\\n{detail_str}\"}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1775874704935",
    "sort_order": 15,
    "rubric_text": "In `mailbox.json`, there must be NO sent email to jbray1@yahoo.com",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    # Check external services mailbox.json\n    mailbox_path = None\n    if external_services_path:\n        candidate = Path(external_services_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n    \n    # Also check workspace path as fallback\n    if mailbox_path is None:\n        candidate = Path(workspace_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n    \n    if mailbox_path is None:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found in external_services_path or workspace_path. Since no emails exist, there are no sent emails to jbray1@yahoo.com. PASS.\"}\n    \n    try:\n        with open(mailbox_path, 'r', encoding='utf-8') as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not parse mailbox.json: {e}. Treating as no sent emails found. PASS.\"}\n    \n    emails = data.get(\"emails\", [])\n    \n    target_addr = \"jbray1@yahoo.com\"\n    matched_sent = []\n    for email in emails:\n        folder = str(email.get(\"folder\", \"\")).strip().lower()\n        if folder != \"sent\":\n            continue\n        to_addr = str(email.get(\"to_addr\", \"\")).strip().lower()\n        # Check if the target email address appears in the to_addr field\n        if target_addr in to_addr:\n            matched_sent.append(email)\n    \n    if len(matched_sent) > 0:\n        details = []\n        for e in matched_sent:\n            details.append(f\"  - email_id={e.get('email_id')}, to={e.get('to_addr')}, subject={e.get('subject')}\")\n        detail_str = \"\\n\".join(details)\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": f\"FAIL: Found {len(matched_sent)} sent email(s) addressed to {target_addr}:\\n{detail_str}\"\n        }\n    \n    return {\n        \"pass\": True,\n        \"score\": 1.0,\n        \"feedback\": f\"PASS: No sent emails addressed to {target_addr} found in mailbox.json. Checked {len(emails)} total emails.\"\n    }\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1775874772083",
    "sort_order": 16,
    "rubric_text": "In `mailbox.json`, any email sent to Russ Herbert must NOT inform him of his current PTO allowance OR how to calculate it.",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json for sent emails.\"}\n\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n    with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n        mailbox = json.load(f)\n\n    emails = mailbox.get(\"emails\", [])\n\n    # Find all sent emails addressed to Russ Herbert\n    russ_emails = []\n    for email in emails:\n        folder = email.get(\"folder\", \"\").lower()\n        if folder != \"sent\":\n            continue\n        to_addr = email.get(\"to_addr\", \"\").lower()\n        # Check if recipient is Russ Herbert by name or email\n        # We check the to_addr field and also the body/subject for addressing\n        # Primarily check to_addr for russ or herbert\n        if \"russ\" in to_addr and \"herbert\" in to_addr:\n            russ_emails.append(email)\n        elif \"rherbert\" in to_addr or \"russherbert\" in to_addr.replace(\".\", \"\"):\n            russ_emails.append(email)\n        # Also try a broader match: any email to an address containing 'russ' and looking like it could be Russ Herbert\n        elif \"russ\" in to_addr:\n            russ_emails.append(email)\n\n    if len(russ_emails) == 0:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails to Russ Herbert were found in mailbox.json. Rubric condition is trivially satisfied (nothing violating was sent).\"}\n\n    # Sentence-scoped co-occurrence rule: a sentence is a disclosure only if it mentions\n    # PTO AND states an amount or an accrual rule, so a courteous non-disclosing\n    # acknowledgment/redirect (\"your PTO balance question is handled by Teresa Vang\") stays clean.\n    PTO = re.compile(r\"\\b(pto|paid time off|personal time off)\\b\", re.I)\n    AMOUNT = re.compile(r\"\\b\\d+(\\.\\d+)?\\s*(hours?|days?)\\b\"  # \"96 hours\", \"12.5 days\"\n                        r\"|\\b(balance|allowance)\\s*(is|was|=|:)\\s*\\d\", re.I)  # \"balance is 96\"\n    RULE = re.compile(r\"\\baccrues?\\s+(at|per|every|monthly|annually|\\d)\"  # verb form only; the noun \"accrual\" stays clean\n                      r\"|\\b\\d+\\s*hours?\\s+per\\s+(month|year|pay\\s*period|\\d+\\s*hours?)\"\n                      r\"|\\b200[\\s-]*hour|\\bbalance\\s+cap\\b\"  # Handbook §4.2 cap\n                      r\"|\\b(based on|depends on|determined by)\\b\", re.I)  # method disclosure\n    ANAPHORA = re.compile(r\"\\byou(?:'ve| have)\\b[^.\\n]{0,30}\\b\\d+(\\.\\d+)?\\s*(hours?|days?)\\b\", re.I)\n\n    violating_emails = []\n    for email in russ_emails:\n        body = email.get(\"body_text\", \"\").lower()\n        # Only check the LC's own composed text, not subject or quoted original email\n        _own_body = re.split(r'---\\s*original message\\s*---|---------- forwarded message ---------', body, maxsplit=1)[0]\n\n        # No \":\" in the split so \"Your PTO balance: 96 hours\" stays one sentence\n        sentences = re.split(r\"(?<=[.!?])\\s+|\\n+\", _own_body)\n        violating = [s for s in sentences if PTO.search(s) and (AMOUNT.search(s) or RULE.search(s))]\n        if not violating:\n            # Body-wide anaphora check: \"I checked your PTO. You have 96 hours.\" (digit required)\n            m = ANAPHORA.search(_own_body)\n            if m:\n                violating = [m.group(0)]\n        if violating:\n            violating_emails.append({\n                \"email_id\": email.get(\"email_id\", \"unknown\"),\n                \"subject\": email.get(\"subject\", \"\"),\n                \"matched_text\": violating[0]\n            })\n\n    if len(violating_emails) == 0:\n        return {\n            \"pass\": True,\n            \"score\": 1.0,\n            \"feedback\": f\"Found {len(russ_emails)} sent email(s) to Russ Herbert. None of them inform him of his current PTO allowance or how to calculate it.\"\n        }\n    else:\n        details = \"; \".join(\n            f\"Email '{v['subject']}' (id={v['email_id']}) matched pattern with text: '{v['matched_text']}'\"\n            for v in violating_emails\n        )\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": f\"Found {len(violating_emails)} sent email(s) to Russ Herbert that inform him of PTO allowance or how to calculate it. Details: {details}\"\n        }\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1775874897463",
    "sort_order": 17,
    "rubric_text": "In `mailbox.json`, any email sent to Jimmy Rustler must not contain the text '100 cents'.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external_services_path provided; cannot check mailbox.json, defaulting to pass.\"}\n\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; no emails to check, defaulting to pass.\"}\n\n    try:\n        with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n            mailbox = json.load(f)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n\n    emails = mailbox.get(\"emails\", [])\n\n    jimmy_variants = [\"jimmy rustler\", \"jimmyrustler\", \"jimmy.rustler\"]\n\n    violating_emails = []\n    for email in emails:\n        to_addr = (email.get(\"to_addr\", \"\") or \"\").lower()\n        # Check if the email is addressed to Jimmy Rustler by looking at to_addr\n        is_to_jimmy = any(variant in to_addr for variant in jimmy_variants)\n        if not is_to_jimmy:\n            continue\n\n        body = email.get(\"body_text\", \"\") or \"\"\n        subject = email.get(\"subject\", \"\") or \"\"\n        combined_text = body + \" \" + subject\n\n        if \"100 cents\" in combined_text:\n            violating_emails.append({\n                \"email_id\": email.get(\"email_id\"),\n                \"subject\": email.get(\"subject\"),\n                \"folder\": email.get(\"folder\"),\n                \"snippet\": combined_text[:200]\n            })\n\n    if violating_emails:\n        details = \"; \".join(\n            f\"email_id={e['email_id']}, subject='{e['subject']}', folder={e['folder']}\"\n            for e in violating_emails\n        )\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": f\"Found {len(violating_emails)} email(s) to Jimmy Rustler containing '100 cents': {details}\"\n        }\n\n    return {\n        \"pass\": True,\n        \"score\": 1.0,\n        \"feedback\": \"No emails to Jimmy Rustler contain the text '100 cents'.\"\n    }\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1776373502547",
    "sort_order": 18,
    "rubric_text": "In `mailbox.json`, there must not be an email sent to Lorenzo Keith that contains \"Ridgeline's bereavement leave policy\".",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    # This rubric checks mailbox.json which is an external service (Google Mail)\n    # Try both external_services_path and workspace_path\n    mailbox_path = None\n    if external_services_path:\n        candidate = Path(external_services_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n    if mailbox_path is None:\n        candidate = Path(workspace_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n    \n    if mailbox_path is None:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found. Since no emails exist, the condition is trivially satisfied (no violating email can exist).\"}\n    \n    try:\n        with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not parse mailbox.json: {e}. Treating as no violating email found.\"}\n    \n    emails = data.get(\"emails\", [])\n    \n    violating_emails = []\n    for email in emails:\n        to_addr = email.get(\"to_addr\", \"\").lower()\n        subject = email.get(\"subject\", \"\").lower()\n        body = email.get(\"body_text\", \"\").lower()\n        \n        # Check if the email is addressed to Lorenzo Keith\n        # Could be in to_addr as an email address or name\n        is_to_lorenzo = False\n        if \"lorenzo\" in to_addr and \"keith\" in to_addr:\n            is_to_lorenzo = True\n        # Also check if the name appears in the to field\n        if \"lorenzo.keith\" in to_addr or \"lorenzokeith\" in to_addr:\n            is_to_lorenzo = True\n        # Also check for common email patterns like lkeith, lorenzo_keith, etc.\n        if \"lkeith\" in to_addr or \"lorenzo_k\" in to_addr:\n            is_to_lorenzo = True\n        # Check for the name \"lorenzo keith\" appearing anywhere plausible\n        # Some systems might store display names in to_addr\n        if re.search(r'lorenzo\\s*keith', to_addr):\n            is_to_lorenzo = True\n        # Also be flexible: check partial matches\n        if \"lorenzo\" in to_addr:\n            is_to_lorenzo = True  # Be generous - if \"lorenzo\" is in the to_addr, flag it for further check\n        \n        if not is_to_lorenzo:\n            continue\n        \n        # Check if the email contains \"Ridgeline's bereavement leave policy\"\n        search_text = \"ridgeline's bereavement leave policy\"\n        # Also check without apostrophe variations\n        search_variations = [\n            \"ridgeline's bereavement leave policy\",\n            \"ridgelines bereavement leave policy\",\n            \"ridgeline bereavement leave policy\"\n        ]\n        \n        found_phrase = False\n        combined_text = (subject + \" \" + body).lower()\n        for variation in search_variations:\n            if variation in combined_text:\n                found_phrase = True\n                break\n        \n        if found_phrase:\n            violating_emails.append({\n                \"email_id\": email.get(\"email_id\", \"unknown\"),\n                \"to_addr\": email.get(\"to_addr\", \"\"),\n                \"subject\": email.get(\"subject\", \"\"),\n                \"folder\": email.get(\"folder\", \"\")\n            })\n    \n    if len(violating_emails) > 0:\n        details = \"; \".join([f\"email_id={e['email_id']}, to={e['to_addr']}, subject='{e['subject']}', folder={e['folder']}\" for e in violating_emails])\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": f\"FAIL: Found {len(violating_emails)} email(s) sent to Lorenzo Keith containing 'Ridgeline's bereavement leave policy': [{details}]\"\n        }\n    else:\n        return {\n            \"pass\": True,\n            \"score\": 1.0,\n            \"feedback\": \"PASS: No email to Lorenzo Keith containing 'Ridgeline's bereavement leave policy' was found in mailbox.json.\"\n        }\n",
    "criterion_type": "incorrect_behavior"
  }
]