handbook/tasks/hr_crestwood_university_da394eac/tests/rubrics.json

[
  {
    "id": "e9e30a9b-fdae-4d1a-b35a-bce846762f57",
    "sort_order": 0,
    "rubric_text": "In `mailbox.json`, no sent emails must exist with `to_addr` set to `facilities@crestwood.edu`",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n    violations = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\" and \"facilities@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n    if violations:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} sent email(s) to facilities@crestwood.edu, expected none.\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent emails to facilities@crestwood.edu found.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "e7bfa402-fa2a-4fc5-bdf6-b5099fa59d0a",
    "sort_order": 1,
    "rubric_text": "In `mailbox.json`, the email from `facilities@crestwood.edu` must be in the `general` folder",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n    from_facilities = [e for e in emails if \"facilities@crestwood.edu\" in str(e.get(\"from_addr\", \"\")).lower()]\n    if not from_facilities:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email from facilities@crestwood.edu found.\"}\n    for e in from_facilities:\n        folder = e.get(\"folder\", \"\").lower()\n        if \"general\" not in folder:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email from facilities@crestwood.edu is in folder '{e.get('folder')}', expected 'general'.\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email from facilities@crestwood.edu is in the general folder.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "e7f91235-98e4-4389-95cf-2b7767dea2f1",
    "sort_order": 2,
    "rubric_text": "In `slack_data.json`, channel `#hr-general` must contain a message with text `[OUT OF SCOPE] - HR office light bulb out – Main Campus rm 214`",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    with open(slack_path) as f:\n        data = json.load(f)\n    channels = data.get(\"channels\", {})\n    hr_general_id = None\n    for cid, cdata in channels.items():\n        if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-general\":\n            hr_general_id = cid\n            break\n    if hr_general_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-general not found.\"}\n    messages = data.get(\"messages\", {}).get(hr_general_id, [])\n\n    def normalize_dashes(s):\n        # Collapse any hyphen/en-dash/em-dash/minus to a single canonical '-'\n        return (s.replace(\"\\u2014\", \"-\")\n                 .replace(\"\\u2013\", \"-\")\n                 .replace(\"\\u2212\", \"-\")\n                 .replace(\"-\", \"-\"))\n\n    target = normalize_dashes(\"[OUT OF SCOPE] - HR office light bulb out - Main Campus rm 214\").lower()\n    for msg in messages:\n        text = normalize_dashes(msg.get(\"text\", \"\")).lower()\n        if target in text:\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required message in #hr-general: '{msg.get('text')}'\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Message '[OUT OF SCOPE] - HR office light bulb out - Main Campus rm 214' (any dash variant) not found in #hr-general. Messages found: {[m.get('text') for m in messages]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "6059ef2b-9c84-4307-8785-3c9e1948af51",
    "sort_order": 3,
    "rubric_text": "In `mailbox.json`, the email from `l.fontaine@crestwood.edu` must be in the `leave` folder",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n    from_fontaine = [e for e in emails if \"l.fontaine@crestwood.edu\" in str(e.get(\"from_addr\", \"\")).lower()]\n    if not from_fontaine:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email from l.fontaine@crestwood.edu found.\"}\n    for e in from_fontaine:\n        folder = e.get(\"folder\", \"\").lower()\n        if \"leave\" not in folder:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email from l.fontaine@crestwood.edu is in folder '{e.get('folder')}', expected 'leave'.\"}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email from l.fontaine@crestwood.edu is in the leave folder.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "fa6332bb-e59b-433e-b135-335bcb794eef",
    "sort_order": 4,
    "rubric_text": "In `mailbox.json`, a sent email to `l.fontaine@crestwood.edu` must exist with body containing PTO balance of 68 hours, accrual rate of 4.62 hours per pay period, and contact info for Patricia Huang at p.huang@crestwood.edu",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n    candidates = [e for e in emails if \"l.fontaine@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n    if not candidates:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to l.fontaine@crestwood.edu found.\"}\n    required_phrases = [\"68 hours\", \"4.62\", \"p.huang@crestwood.edu\", \"04/07/2026\"]\n    for e in candidates:\n        body = e.get(\"body_text\", \"\").lower()\n        if all(phrase.lower() in body for phrase in required_phrases):\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Email to l.fontaine@crestwood.edu found with required PTO balance content.\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email to l.fontaine@crestwood.edu found but missing required content. Required: {required_phrases}. Bodies found: {[e.get('body_text','')[:200] for e in candidates]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "4ed6f0e7-1d75-45cc-a70d-a53741123f63",
    "sort_order": 5,
    "rubric_text": "In `mailbox.json`, a sent email to `p.huang@crestwood.edu` with CC `t.salazar@crestwood.edu` with subject `I-9 Urgent Reminder — Salazar, CU-1026` must exist with body mentioning Tomás Salazar, CU-1026, 04/08/2026 federal deadline, and Section 2 of I-9",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n\n    # Find emails sent to p.huang@crestwood.edu\n    candidates = []\n    for e in emails:\n        to_addr = e.get(\"to_addr\", \"\") or e.get(\"to\", \"\")\n        if isinstance(to_addr, list):\n            to_lower = [a.lower() for a in to_addr]\n        else:\n            to_lower = [str(to_addr).lower()]\n        if any(\"p.huang@crestwood.edu\" in a for a in to_lower):\n            candidates.append(e)\n\n    if not candidates:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to p.huang@crestwood.edu found.\"}\n\n    # Filter for I-9 / Salazar / CU-1026 related subject\n    salazar_emails = []\n    for e in candidates:\n        subj = e.get(\"subject\", \"\").lower()\n        if \"salazar\" in subj or \"cu-1026\" in subj or \"i-9\" in subj or \"i9\" in subj:\n            salazar_emails.append(e)\n\n    if not salazar_emails:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No I-9/Salazar-related email found among emails to p.huang@crestwood.edu. Subjects found: {[e.get('subject') for e in candidates]}\"}\n\n    for e in salazar_emails:\n        body = e.get(\"body_text\", \"\") or e.get(\"body\", \"\") or \"\"\n        subject = e.get(\"subject\", \"\")\n        full_text = (subject + \" \" + body).lower()\n\n        # Check CC includes t.salazar@crestwood.edu\n        cc = e.get(\"cc_addr\", \"\") or e.get(\"cc\", \"\") or \"\"\n        if isinstance(cc, list):\n            cc_lower = [a.lower() for a in cc]\n        else:\n            cc_lower = [str(cc).lower()]\n        has_cc = any(\"t.salazar@crestwood.edu\" in c for c in cc_lower)\n\n        # Check body mentions: Salazar (or Tomás Salazar), CU-1026, April 8 2026 deadline, Section 2\n        has_salazar = \"salazar\" in full_text\n        has_cu1026 = \"cu-1026\" in full_text or \"cu1026\" in full_text\n\n        # Accept various date formats for April 8, 2026\n        date_patterns = [\n            \"04/08/2026\", \"4/8/2026\",\n        ]\n        has_date = any(p in full_text for p in date_patterns)\n\n        # Section 2 check - be lenient\n        has_section2 = \"section 2\" in full_text or \"section ii\" in full_text or \"section two\" in full_text or \"section-2\" in full_text\n\n        if has_salazar and has_cu1026 and has_date and has_section2:\n            if not has_cc:\n                return {\"pass\": True, \"score\": 0.8, \"feedback\": f\"Found I-9 urgent reminder email to p.huang@crestwood.edu for Salazar CU-1026 with required content, but CC to t.salazar@crestwood.edu not confirmed. CC field: {cc}\"}\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found I-9 urgent reminder email to p.huang@crestwood.edu with CC t.salazar@crestwood.edu for Salazar CU-1026 with all required content (Tomás Salazar, CU-1026, April 8 2026 deadline, Section 2).\"}\n\n    # Provide detailed failure info\n    details = []\n    for e in salazar_emails:\n        body = e.get(\"body_text\", \"\") or e.get(\"body\", \"\") or \"\"\n        full_text = (e.get(\"subject\", \"\") + \" \" + body).lower()\n        cc = e.get(\"cc_addr\", \"\") or e.get(\"cc\", \"\") or \"\"\n        missing = []\n        if \"salazar\" not in full_text:\n            missing.append(\"Salazar\")\n        if \"cu-1026\" not in full_text and \"cu1026\" not in full_text:\n            missing.append(\"CU-1026\")\n        date_patterns = [\n            \"april 8\", \"april 08\", \"04/08/2026\", \"4/8/2026\",\n            \"2026-04-08\", \"apr 8\", \"apr 08\", \"april 8, 2026\",\n            \"april 8th\", \"april 8th, 2026\", \"04-08-2026\",\n            \"4-8-2026\", \"april 08, 2026\"\n        ]\n        if not any(p in full_text for p in date_patterns):\n            missing.append(\"April 8 2026 date\")\n        if \"section 2\" not in full_text and \"section ii\" not in full_text and \"section two\" not in full_text and \"section-2\" not in full_text:\n            missing.append(\"Section 2\")\n        details.append({\"subject\": e.get(\"subject\"), \"cc\": cc, \"missing\": missing, \"body_snippet\": full_text[:300]})\n\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"I-9 urgent reminder email found but missing required content. Details: {details}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "559efc43-ae89-4956-8045-90ca9cb46fbb",
    "sort_order": 6,
    "rubric_text": "In `benefits_enrollment.xlsx`, cell C26 must be set to `Default Enrollment` and cell D26 must be set to `05/01/2026`",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    wb_path = Path(workspace_path) / \"benefits_enrollment.xlsx\"\n    if not wb_path.exists():\n        # Try searching\n        matches = list(Path(workspace_path).glob(\"**/benefits_enrollment.xlsx\"))\n        if not matches:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": \"benefits_enrollment.xlsx not found in workspace.\"}\n        wb_path = matches[0]\n    wb = openpyxl.load_workbook(wb_path)\n    ws = wb.active\n    c26 = ws[\"C26\"].value\n    d26 = ws[\"D26\"].value\n    c26_str = str(c26).strip() if c26 is not None else \"\"\n    d26_str = str(d26).strip() if d26 is not None else \"\"\n    c26_ok = \"default enrollment\" in c26_str.lower()\n    d26_ok = \"05/01/2026\" in d26_str or d26_str == \"05/01/2026\"\n    # Also check if d26 is a date object\n    import datetime\n    if hasattr(d26, 'month') and hasattr(d26, 'year'):\n        d26_ok = d26.month == 5 and d26.year == 2026 and d26.day == 1\n    if c26_ok and d26_ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"C26='{c26}' (Default Enrollment) and D26='{d26}' (05/01/2026) correctly set.\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"C26='{c26}' (expected 'Default Enrollment', ok={c26_ok}), D26='{d26}' (expected 05/01/2026, ok={d26_ok}).\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "4323ac49-9f39-4121-9b6d-13ab81d30ed0",
    "sort_order": 7,
    "rubric_text": "In `onboarding_checklist.xlsx`, cell E15 must be set to `05/01/2026` (Benefits Enrolled date)",
    "verifier_code": "from pathlib import Path\nimport openpyxl\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n    wb_path = Path(workspace_path) / \"onboarding_checklist.xlsx\"\n    if not wb_path.exists():\n        matches = list(Path(workspace_path).glob(\"**/onboarding_checklist.xlsx\"))\n        if not matches:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": \"onboarding_checklist.xlsx not found in workspace.\"}\n        wb_path = matches[0]\n    wb = openpyxl.load_workbook(wb_path)\n    ws = wb.active\n    e15 = ws[\"E15\"].value\n    e15_str = str(e15).strip() if e15 is not None else \"\"\n    ok = \"05/01/2026\" in e15_str\n    if hasattr(e15, 'month') and hasattr(e15, 'year'):\n        ok = e15.month == 5 and e15.year == 2026 and e15.day == 1\n    if ok:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"E15='{e15}' correctly set to 05/01/2026 for Benefits Enrolled.\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"E15='{e15}' does not match expected 05/01/2026.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "f2b2962e-5acc-4346-a75d-c52075fa9e35",
    "sort_order": 8,
    "rubric_text": "In `employee_roster.xlsx`, cell L23 must be set to `Active`",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    wb_path = Path(workspace_path) / \"employee_roster.xlsx\"\n    if not wb_path.exists():\n        matches = list(Path(workspace_path).glob(\"**/employee_roster.xlsx\"))\n        if not matches:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": \"employee_roster.xlsx not found in workspace.\"}\n        wb_path = matches[0]\n    wb = openpyxl.load_workbook(wb_path)\n    ws = wb.active\n    l23 = ws[\"L23\"].value\n    l23_str = str(l23).strip() if l23 is not None else \"\"\n    if l23_str.lower() == \"active\":\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"L23='{l23}' correctly set to Active.\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"L23='{l23}' does not match expected 'Active'.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "efd47f2e-f94d-4e44-b54b-7b67ad1ba7fe",
    "sort_order": 9,
    "rubric_text": "In `slack_data.json`, channel `#hr-onboarding` must contain a message with exact text `ONBOARDING COMPLETE: William Trent, CU-1022, 04/07/2026`",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    with open(slack_path) as f:\n        data = json.load(f)\n    channels = data.get(\"channels\", {})\n    hr_onboarding_id = None\n    for cid, cdata in channels.items():\n        if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-onboarding\":\n            hr_onboarding_id = cid\n            break\n    if hr_onboarding_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-onboarding not found.\"}\n    messages = data.get(\"messages\", {}).get(hr_onboarding_id, [])\n    target = \"ONBOARDING COMPLETE: William Trent, CU-1022, 04/07/2026\"\n    for msg in messages:\n        if target.lower() in msg.get(\"text\", \"\").lower():\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required message in #hr-onboarding: '{msg.get('text')}'\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Message '{target}' not found in #hr-onboarding. Messages: {[m.get('text') for m in messages]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "62bb6384-ef1c-4030-b4d3-3dd3ff76d725",
    "sort_order": 10,
    "rubric_text": "In `mailbox.json`, a sent email to `w.trent@crestwood.edu` with subject `Default Benefits Enrollment Confirmation — Trent, CU-1022` must exist with body mentioning default enrollment, 04/02/2026 deadline, and 05/01/2026 effective date",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n    candidates = [e for e in emails if \"w.trent@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n    if not candidates:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to w.trent@crestwood.edu found.\"}\n    subject_match = [e for e in candidates if \"cu-1022\" in e.get(\"subject\", \"\").lower() or \"trent\" in e.get(\"subject\", \"\").lower()]\n    required_phrases = [\"04/02/2026\", \"05/01/2026\", \"default\"]\n    for e in subject_match:\n        body = e.get(\"body_text\", \"\")\n        if all(phrase.lower() in body.lower() for phrase in required_phrases):\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found default benefits enrollment confirmation email to w.trent@crestwood.edu with required content.\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Default benefits enrollment confirmation email to w.trent@crestwood.edu not found or missing required content. Required: {required_phrases}. Subjects: {[e.get('subject') for e in candidates]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "97c5f3d5-b336-4a10-b782-d15d710b62d8",
    "sort_order": 11,
    "rubric_text": "In `mailbox.json`, a sent email to `p.huang@crestwood.edu` with subject `FMLA Deadline Passed — CU-1023` must exist with body mentioning Meghan Cross FMLA Certification overdue",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n    candidates = [e for e in emails if \"p.huang@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n    if not candidates:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to p.huang@crestwood.edu found.\"}\n    fmla_emails = [e for e in candidates if \"fmla\" in e.get(\"subject\", \"\").lower() and \"cu-1023\" in e.get(\"subject\", \"\").lower()]\n    if not fmla_emails:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No FMLA CU-1023 email to p.huang@crestwood.edu found. Subjects: {[e.get('subject') for e in candidates]}\"}\n    for e in fmla_emails:\n        body = e.get(\"body_text\", \"\")\n        if \"cross\" in body.lower() or \"meghan\" in body.lower():\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found FMLA Deadline Passed email to p.huang@crestwood.edu for CU-1023 (Meghan Cross).\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FMLA CU-1023 email to p.huang@crestwood.edu found but body missing 'Cross'/'Meghan'. Bodies: {[e.get('body_text','')[:200] for e in fmla_emails]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "342775db-78be-4cd8-a801-e523a0286686",
    "sort_order": 12,
    "rubric_text": "In `mailbox.json`, a sent email to `m.cross@crestwood.edu` with subject `FMLA Certification Overdue — Cross, CU-1023` OR `Template 18 — Cross, CU-1023` OR `Template 18 – FMLA Certification Overdue — Cross, CU-1023` must exist with body mentioning due date 04/07/2026, FMLA Medical Certification Form, and contact info for Patricia Huang",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n    # Find emails sent to m.cross@crestwood.edu\n    candidates = [e for e in emails if \"m.cross@crestwood.edu\" in str(e.get(\"to_addr\", \"\")).lower()]\n    if not candidates:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to m.cross@crestwood.edu found.\"}\n\n    # Check subject matches one of the allowed patterns (loose matching)\n    subject_keywords = [\"fmla\", \"template 18\", \"cross\", \"cu-1023\"]\n\n    # Check body for required content\n    # Due date: 04/07/2026 in various formats\n    date_patterns = [\"04/07/2026\", \"4/7/2026\"]\n    # FMLA Medical Certification Form\n    fmla_form_patterns = [\"fmla medical certification\", \"fmla certification\", \"medical certification form\"]\n    # Contact info for Patricia Huang - could be email, phone, or just name\n    patricia_patterns = [\"patricia huang\", \"p.huang@crestwood.edu\", \"p. huang\"]\n\n    for e in candidates:\n        body = (e.get(\"body_text\", \"\") or \"\").lower()\n        subject = (e.get(\"subject\", \"\") or \"\").lower()\n        combined = subject + \" \" + body\n\n        # Subject check: must relate to FMLA or Template 18 and Cross/CU-1023\n        subject_ok = False\n        if (\"fmla\" in subject or \"template 18\" in subject) and (\"cross\" in subject or \"cu-1023\" in subject):\n            subject_ok = True\n\n        if not subject_ok:\n            continue\n\n        # Date check\n        date_ok = any(dp in combined for dp in date_patterns)\n\n        # FMLA form check\n        fmla_form_ok = any(fp in combined for fp in fmla_form_patterns)\n\n        # Patricia Huang contact info check\n        patricia_ok = any(pp in combined for pp in patricia_patterns)\n\n        if date_ok and fmla_form_ok and patricia_ok:\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found FMLA Certification Overdue email to m.cross@crestwood.edu with required content. Subject: {e.get('subject')}\"}\n\n        # Collect what's missing for feedback\n        missing = []\n        if not date_ok:\n            missing.append(\"due date (04/07/2026)\")\n        if not fmla_form_ok:\n            missing.append(\"FMLA Medical Certification Form reference\")\n        if not patricia_ok:\n            missing.append(\"Patricia Huang contact info\")\n\n    # If we get here, no email matched fully\n    all_subjects = [e.get('subject', '') for e in candidates]\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FMLA Certification Overdue email to m.cross@crestwood.edu not found or missing required content. Need: due date 04/07/2026, FMLA Medical Certification Form mention, and Patricia Huang contact info. Candidate subjects: {all_subjects}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "543a04a0-b8fa-4f83-96f1-70823791a25d",
    "sort_order": 13,
    "rubric_text": "In `slack_data.json`, channel `#hr-leave-requests` must contain a message with text `FMLA CERTIFICATION OVERDUE: Meghan Cross, CU-1023, Due: 04/07/2026` and mentioning overdue",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    with open(slack_path) as f:\n        data = json.load(f)\n    channels = data.get(\"channels\", {})\n    hr_leave_id = None\n    for cid, cdata in channels.items():\n        if cdata.get(\"name\", \"\").lower().strip(\"#\") == \"hr-leave-requests\":\n            hr_leave_id = cid\n            break\n    if hr_leave_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #hr-leave-requests not found.\"}\n    messages = data.get(\"messages\", {}).get(hr_leave_id, [])\n    required_phrases = [\"FMLA CERTIFICATION OVERDUE\", \"Meghan Cross\", \"CU-1023\", \"04/07/2026\"]\n    for msg in messages:\n        text = msg.get(\"text\", \"\")\n        if all(phrase.lower() in text.lower() for phrase in required_phrases):\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found required FMLA overdue message in #hr-leave-requests: '{text}'\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Required FMLA overdue message not found in #hr-leave-requests. Required phrases: {required_phrases}. Messages: {[m.get('text') for m in messages]}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "6b40093b-ab7a-46b8-9129-bcd2c58d223a",
    "sort_order": 14,
    "rubric_text": "In `jira_state.json`, issue `HR-7` must have comments containing all required entries for steps 11.3 and 11.4, and description appended with `Status — CLOSED`",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n    jira_path = Path(external_services_path) / \"jira_state.json\"\n    if not jira_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n    with open(jira_path) as f:\n        data = json.load(f)\n\n    def adf_to_text(node):\n        \"\"\"Flatten an Atlassian Document Format node (dict/list) to plain text. Passthrough for strings.\"\"\"\n        if node is None:\n            return \"\"\n        if isinstance(node, str):\n            return node\n        out = []\n        def walk(n):\n            if isinstance(n, dict):\n                if n.get(\"type\") == \"text\":\n                    out.append(n.get(\"text\", \"\"))\n                for c in n.get(\"content\", []) or []:\n                    walk(c)\n                if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n                    out.append(\"\\n\")\n            elif isinstance(n, list):\n                for x in n:\n                    walk(x)\n        walk(node)\n        return \"\".join(out).strip()\n\n    # Navigate to issues\n    issues = data.get(\"issues\", data) if isinstance(data, dict) else data\n\n    trent_issue = None\n    matched_key = None\n    if isinstance(issues, dict):\n        for key, val in issues.items():\n            if \"trent\" in key.lower() and \"onboard\" in key.lower():\n                trent_issue = val if isinstance(val, dict) else {\"raw\": val}\n                matched_key = key\n                break\n        if trent_issue is None:\n            for key, val in issues.items():\n                combined = (key + \" \" + json.dumps(val, default=str)).lower()\n                if \"trent\" in combined and \"onboard\" in combined:\n                    trent_issue = val if isinstance(val, dict) else {\"raw\": val}\n                    matched_key = key\n                    break\n    elif isinstance(issues, list):\n        for item in issues:\n            if isinstance(item, dict):\n                item_str = json.dumps(item, default=str).lower()\n                if \"trent\" in item_str and \"onboard\" in item_str:\n                    trent_issue = item\n                    matched_key = item.get(\"key\", \"list_item\")\n                    break\n\n    if trent_issue is None:\n        all_keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A (list)\"\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-7 not found in jira_state.json. Available keys: {all_keys}\"}\n\n    # Gather comments. Handle both schemas:\n    #   (a) comments nested inside the issue: issue[\"comments\"] = [...]\n    #   (b) comments stored in a parallel top-level dict: data[\"comments\"][issue_key] = [...]\n    # Each comment's body may be a plain string OR an ADF dict.\n    comments_raw = []\n    nested = trent_issue.get(\"comments\")\n    if isinstance(nested, list):\n        comments_raw.extend(nested)\n    if isinstance(data, dict):\n        parallel = data.get(\"comments\", {})\n        if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n            comments_raw.extend(parallel[matched_key])\n\n    comments_text_parts = []\n    for c in comments_raw:\n        if isinstance(c, dict):\n            for field in (\"body\", \"text\", \"content\"):\n                if field in c:\n                    comments_text_parts.append(adf_to_text(c[field]))\n        else:\n            comments_text_parts.append(str(c))\n    comments_text = \" \".join(comments_text_parts)\n    comments_text_lower = comments_text.lower()\n\n    # Description. Handle both schemas:\n    #   (a) issue[\"description\"] directly (string or ADF)\n    #   (b) issue[\"fields\"][\"description\"] (string or ADF)\n    description = trent_issue.get(\"description\")\n    if description in (None, \"\") and isinstance(trent_issue.get(\"fields\"), dict):\n        description = trent_issue[\"fields\"].get(\"description\")\n    description_text = adf_to_text(description)\n    desc_lower = description_text.lower()\n\n    all_text_lower = comments_text_lower + \" \" + desc_lower\n\n    required_comment_fragments = [\n        (\"11.3\", [\"11.3\", \"step 11.3\", \"11-3\"]),\n        (\"benefits_enrollment\", [\"benefits_enrollment\", \"benefits enrollment\", \"benefits-enrollment\"]),\n        (\"Default Enrollment\", [\"default enrollment\", \"default enroll\"]),\n        (\"05/01/2026\", [\"05/01/2026\", \"5/1/2026\"]),\n        (\"Template 10\", [\"template 10\", \"template-10\", \"template_10\"]),\n        (\"w.trent@crestwood.edu\", [\"w.trent@crestwood\", \"trent@crestwood\"]),\n        (\"11.4\", [\"11.4\", \"step 11.4\", \"11-4\"]),\n        (\"employee_roster\", [\"employee_roster\", \"employee roster\", \"employee-roster\"]),\n        (\"Active\", [\"active\"]),\n        (\"#hr-onboarding\", [\"hr-onboarding\", \"hr onboarding\", \"#hr-onboarding\"]),\n    ]\n\n    missing = []\n    for label, variants in required_comment_fragments:\n        if not any(v.lower() in all_text_lower for v in variants):\n            missing.append(label)\n\n    closed_patterns = [\n        \"status — closed\",\n        \"status – closed\",\n        \"status - closed\",\n        \"status -- closed\",\n        \"status—closed\",\n        \"status–closed\",\n        \"status-closed\",\n        \"status: closed\",\n        \"status:closed\",\n    ]\n    closed_in_desc = any(pat in desc_lower for pat in closed_patterns)\n    closed_anywhere = \"closed\" in all_text_lower\n\n    feedback_parts = []\n    score = 1.0\n    passed = True\n\n    if missing:\n        passed = False\n        score = max(0.0, 1.0 - len(missing) * 0.1)\n        feedback_parts.append(f\"Missing comment/description fragments: {missing}\")\n\n    if not closed_in_desc:\n        if closed_anywhere:\n            feedback_parts.append(f\"'CLOSED' found in comments but not clearly in description. Description: '{description_text[:200]}'\")\n            passed = False\n            score = min(score, 0.5)\n        else:\n            passed = False\n            score = 0.0\n            feedback_parts.append(f\"Description does not contain 'Status – CLOSED' annotation. Description: '{description_text[:300]}'\")\n\n    if passed:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"HR-7 (key: {matched_key}) has all required comment entries for 11.3 and 11.4, and description contains CLOSED status.\"}\n    return {\"pass\": False, \"score\": score, \"feedback\": f\"Issue key: {matched_key}. Found {len(comments_raw)} comments. \" + \" | \".join(feedback_parts)}",
    "criterion_type": "expected_output"
  },
  {
    "id": "fd6fcf96-a025-43ef-b1ec-d6bd8023ec7d",
    "sort_order": 15,
    "rubric_text": "In `jira_state.json`, issue `HR-1` must have comments for Template 18 sent to m.cross@crestwood.edu, slack posted to #hr-leave-requests, email forwarded to p.huang@crestwood.edu, and waiting on Patricia Huang",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n    jira_path = Path(external_services_path) / \"jira_state.json\"\n    if not jira_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n    with open(jira_path) as f:\n        data = json.load(f)\n\n    def adf_to_text(node):\n        if node is None:\n            return \"\"\n        if isinstance(node, str):\n            return node\n        out = []\n        def walk(n):\n            if isinstance(n, dict):\n                if n.get(\"type\") == \"text\":\n                    out.append(n.get(\"text\", \"\"))\n                for c in n.get(\"content\", []) or []:\n                    walk(c)\n                if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n                    out.append(\"\\n\")\n            elif isinstance(n, list):\n                for x in n:\n                    walk(x)\n        walk(node)\n        return \"\".join(out).strip()\n\n    issues = data.get(\"issues\", data) if isinstance(data, dict) and \"issues\" in data else data\n\n    cross_issue = None\n    matched_key = None\n    if isinstance(issues, dict):\n        for key, val in issues.items():\n            if \"cross\" in key.lower() and \"leave\" in key.lower():\n                cross_issue = val if isinstance(val, dict) else {\"raw\": val}\n                matched_key = key\n                break\n    elif isinstance(issues, list):\n        for item in issues:\n            if isinstance(item, dict):\n                s = json.dumps(item, default=str).lower()\n                if \"cross\" in s and \"leave\" in s:\n                    cross_issue = item\n                    matched_key = item.get(\"key\", \"list_item\")\n                    break\n\n    if cross_issue is None:\n        keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A\"\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-1 not found. Keys: {keys}\"}\n\n    comments_raw = []\n    nested = cross_issue.get(\"comments\")\n    if isinstance(nested, list):\n        comments_raw.extend(nested)\n    if isinstance(data, dict):\n        parallel = data.get(\"comments\", {})\n        if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n            comments_raw.extend(parallel[matched_key])\n\n    parts = []\n    for c in comments_raw:\n        if isinstance(c, dict):\n            for field in (\"body\", \"text\", \"content\"):\n                if field in c:\n                    parts.append(adf_to_text(c[field]))\n        else:\n            parts.append(str(c))\n    comments_text = \" \".join(parts)\n    ct = comments_text.lower()\n\n    required_fragments = [\"template 18\", \"m.cross@crestwood.edu\", \"hr-leave-requests\", \"p.huang@crestwood.edu\"]\n    missing = [f for f in required_fragments if f.lower() not in ct]\n    if missing:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"HR-1 (key: {matched_key}) missing fragments: {missing}. Found {len(comments_raw)} comments. Text: '{comments_text[:400]}'\"}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": f\"HR-1 (key: {matched_key}) has required fragments across {len(comments_raw)} comments.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "0b596b96-db7e-445f-a83b-05ef87e66771",
    "sort_order": 16,
    "rubric_text": "In `jira_state.json`, issue `HR-6` must have a comment mentioning Template 6 sent to p.huang@crestwood.edu and step 11.2",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n    jira_path = Path(external_services_path) / \"jira_state.json\"\n    if not jira_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n    with open(jira_path) as f:\n        data = json.load(f)\n\n    def adf_to_text(node):\n        \"\"\"Flatten Atlassian Document Format (dict/list) to plain text; passthrough for strings.\"\"\"\n        if node is None:\n            return \"\"\n        if isinstance(node, str):\n            return node\n        out = []\n        def walk(n):\n            if isinstance(n, dict):\n                if n.get(\"type\") == \"text\":\n                    out.append(n.get(\"text\", \"\"))\n                for c in n.get(\"content\", []) or []:\n                    walk(c)\n                if n.get(\"type\") in (\"paragraph\", \"heading\", \"listItem\"):\n                    out.append(\"\\n\")\n            elif isinstance(n, list):\n                for x in n:\n                    walk(x)\n        walk(node)\n        return \"\".join(out).strip()\n\n    issues = data.get(\"issues\", data) if isinstance(data, dict) and \"issues\" in data else data\n\n    salazar_issue = None\n    matched_key = None\n    if isinstance(issues, dict):\n        for key, val in issues.items():\n            if \"salazar\" in key.lower() and \"onboard\" in key.lower():\n                salazar_issue = val if isinstance(val, dict) else {\"raw\": val}\n                matched_key = key\n                break\n        if salazar_issue is None:\n            for key, val in issues.items():\n                combined = (key + \" \" + json.dumps(val, default=str)).lower()\n                if \"salazar\" in combined and \"onboard\" in combined:\n                    salazar_issue = val if isinstance(val, dict) else {\"raw\": val}\n                    matched_key = key\n                    break\n    elif isinstance(issues, list):\n        for item in issues:\n            if isinstance(item, dict):\n                s = json.dumps(item, default=str).lower()\n                if \"salazar\" in s and \"onboard\" in s:\n                    salazar_issue = item\n                    matched_key = item.get(\"key\", \"list_item\")\n                    break\n\n    if salazar_issue is None:\n        keys = list(issues.keys()) if isinstance(issues, dict) else \"N/A\"\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue HR-6 not found. Keys: {keys}\"}\n\n    # Collect comments from both possible schemas.\n    comments_raw = []\n    nested = salazar_issue.get(\"comments\")\n    if isinstance(nested, list):\n        comments_raw.extend(nested)\n    if isinstance(data, dict):\n        parallel = data.get(\"comments\", {})\n        if isinstance(parallel, dict) and matched_key in parallel and isinstance(parallel[matched_key], list):\n            comments_raw.extend(parallel[matched_key])\n\n    parts = []\n    for c in comments_raw:\n        if isinstance(c, dict):\n            for field in (\"body\", \"text\", \"content\"):\n                if field in c:\n                    parts.append(adf_to_text(c[field]))\n        else:\n            parts.append(str(c))\n    comments_text = \" \".join(parts)\n    ct = comments_text.lower()\n\n    required_fragments = [\"11.2\", \"Template 6\", \"p.huang@crestwood.edu\"]\n    missing = [f for f in required_fragments if f.lower() not in ct]\n    if missing:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"HR-6 (key: {matched_key}) missing comment fragments: {missing}. Found {len(comments_raw)} comments. Text: '{comments_text[:400]}'\"}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": f\"HR-6 (key: {matched_key}) has required comment for Template 6 to p.huang@crestwood.edu across {len(comments_raw)} entries.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "0968f22e-9afc-4fa9-8baf-f8ed52566e11",
    "sort_order": 17,
    "rubric_text": "In `mailbox.json`, there must be exactly 11 sent emails (emails with folder='Sent')",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    with open(mailbox_path) as f:\n        data = json.load(f)\n    emails = data.get(\"emails\", [])\n    sent = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n    if len(sent) == 11:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 11 sent emails. Subjects: {[e.get('subject') for e in sent]}\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 11 sent emails, found {len(sent)}. Subjects: {[e.get('subject') for e in sent]}\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "9fa6493a-d029-4e8f-a6ab-fdade61b8c30",
    "sort_order": 18,
    "rubric_text": "In `slack_data.json`, each channel must have the exact number of messages: #hr-onboarding=15, #hr-offboarding=1, #hr-leave-requests=13, #hr-performance=2, #hr-recruiting=3, #hr-escalations=0, #hr-general=3",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    with open(slack_path) as f:\n        data = json.load(f)\n    channels = data.get(\"channels\", {})\n    messages = data.get(\"messages\", {})\n    expected = {\n        \"hr-onboarding\": 15,\n        \"hr-offboarding\": 1,\n        \"hr-leave-requests\": 13,\n        \"hr-performance\": 2,\n        \"hr-recruiting\": 3,\n        \"hr-escalations\": 0,\n        \"hr-general\": 3\n    }\n    name_to_id = {}\n    for cid, cdata in channels.items():\n        name = cdata.get(\"name\", \"\").lower().strip(\"#\")\n        name_to_id[name] = cid\n    failures = []\n    for channel_name, expected_count in expected.items():\n        cid = name_to_id.get(channel_name)\n        if cid is None:\n            if expected_count == 0:\n                continue\n            failures.append(f\"#{channel_name} not found (expected {expected_count} messages)\")\n            continue\n        actual_count = len(messages.get(cid, []))\n        if actual_count != expected_count:\n            failures.append(f\"#{channel_name}: expected {expected_count} messages, found {actual_count}\")\n    if failures:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Message count mismatches: \" + \"; \".join(failures)}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All channels have the correct number of messages.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "100b7c72-5bd9-4716-93ac-939303f7d470",
    "sort_order": 19,
    "rubric_text": "In `jira_state.json`, there must be exactly 8 issues",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n    jira_path = Path(external_services_path) / \"jira_state.json\"\n    if not jira_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n    with open(jira_path) as f:\n        data = json.load(f)\n    if isinstance(data, dict) and \"issues\" in data:\n        issues = data[\"issues\"]\n    else:\n        issues = data\n    if isinstance(issues, dict):\n        count = len(issues)\n    elif isinstance(issues, list):\n        count = len(issues)\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Unexpected issues format: {type(issues)}\"}\n    if count == 8:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 8 issues in jira_state.json.\"}\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 8 issues in jira_state.json, found {count}.\"}",
    "criterion_type": "incorrect_behavior"
  }
]