handbook/tasks/hr_ridgeline_gear_co_6950ff2b/tests/rubrics.json

[
  {
    "id": "171ed1a2-8c28-4ca4-89b0-a53660a931f1",
    "sort_order": 0,
    "rubric_text": "In file `leave_tracker.xlsx`, there must be two new rows for EMP-1222 (Portia Nzinga, Supply Chain) with different Case IDs — one for FMLA with status 'Denied' and one for FAMLI with status 'Intake', both with leave type 'Continuous', priority 'P3', start date 2026-05-14, manager 'Theo Mbeki', foreseeable flag, and personal email contact method.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n    leave_tracker = None\n    for f in xlsx_files:\n        if 'leave' in f.name.lower() and 'tracker' in f.name.lower():\n            leave_tracker = f\n            break\n    if leave_tracker is None:\n        for f in xlsx_files:\n            if 'leave' in f.name.lower():\n                leave_tracker = f\n                break\n    if leave_tracker is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'No leave_tracker.xlsx file found in workspace'}\n\n    try:\n        wb = openpyxl.load_workbook(leave_tracker)\n        ws = wb.active\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not open leave tracker: {e}'}\n\n    headers = []\n    data_rows = []\n    for i, row in enumerate(ws.iter_rows(values_only=True)):\n        if i == 0:\n            headers = [str(c).strip().lower() if c else '' for c in row]\n        else:\n            if any(c is not None for c in row):\n                data_rows.append(row)\n\n    def find_col(keywords):\n        for kw in keywords:\n            for idx, h in enumerate(headers):\n                if kw in h:\n                    return idx\n        return None\n\n    emp_col = find_col(['emp', 'employee id', 'employee_id'])\n    name_col = find_col(['name'])\n    dept_col = find_col(['dept', 'department', 'supply'])\n    leave_type_col = find_col(['leave type', 'leave_type', 'type'])\n    status_col = find_col(['status'])\n    priority_col = find_col(['priority', 'p3'])\n    duration_col = find_col(['duration', 'continuous', 'leave duration'])\n    start_col = find_col(['start date', 'start_date', 'date'])\n    manager_col = find_col(['manager', 'mbeki'])\n    foreseeable_col = find_col(['foreseeable', 'foresee'])\n    contact_col = find_col(['contact', 'email type', 'personal'])\n    case_col = find_col(['case id', 'case_id', 'caseid'])\n\n    nzinga_rows = []\n    for row in data_rows:\n        row_str = ' '.join(str(c).lower() for c in row if c is not None)\n        if 'nzinga' in row_str or 'emp-1222' in row_str.replace(' ', '') or '1222' in row_str:\n            nzinga_rows.append(row)\n\n    if len(nzinga_rows) < 2:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Expected at least 2 rows for Portia Nzinga/EMP-1222, found {len(nzinga_rows)}'}\n\n    def get_val(row, col):\n        if col is None or col >= len(row):\n            return ''\n        v = row[col]\n        return str(v).strip().lower() if v is not None else ''\n\n    fmla_row = None\n    famli_row = None\n    for row in nzinga_rows:\n        row_str = ' '.join(str(c).lower() for c in row if c is not None)\n        lt = get_val(row, leave_type_col) if leave_type_col is not None else row_str\n        st = get_val(row, status_col) if status_col is not None else row_str\n        if 'fmla' in lt or 'fmla' in row_str:\n            if 'famli' not in lt:\n                fmla_row = row\n        if 'famli' in lt or 'famli' in row_str:\n            famli_row = row\n\n    if fmla_row is None:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not find FMLA row for Portia Nzinga among rows: {nzinga_rows}'}\n    if famli_row is None:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not find FAMLI row for Portia Nzinga among rows: {nzinga_rows}'}\n\n    fmla_str = ' '.join(str(c).lower() for c in fmla_row if c is not None)\n    famli_str = ' '.join(str(c).lower() for c in famli_row if c is not None)\n\n    issues = []\n\n    # Check FMLA row status = Denied\n    if 'denied' not in fmla_str:\n        issues.append(f'FMLA row missing Denied status. Row: {fmla_row}')\n\n    # Check FAMLI row status = Intake\n    if 'intake' not in famli_str:\n        issues.append(f'FAMLI row missing Intake status. Row: {famli_row}')\n\n    # Check both rows have P3\n    if 'p3' not in fmla_str:\n        issues.append('FMLA row missing P3 priority')\n    if 'p3' not in famli_str:\n        issues.append('FAMLI row missing P3 priority')\n\n    # Check both rows have Continuous\n    if 'continuous' not in fmla_str:\n        issues.append('FMLA row missing Continuous leave type')\n    if 'continuous' not in famli_str:\n        issues.append('FAMLI row missing Continuous leave type')\n\n    # Check start date 2026-05-14\n    date_variants = ['2026-05-14', '05/14/2026', '5/14/2026', '2026-5-14', '05-14-2026']\n    fmla_has_date = any(d in fmla_str for d in date_variants)\n    famli_has_date = any(d in famli_str for d in date_variants)\n    if not fmla_has_date:\n        issues.append(f'FMLA row missing start date 2026-05-14. Row: {fmla_row}')\n    if not famli_has_date:\n        issues.append(f'FAMLI row missing start date 2026-05-14. Row: {famli_row}')\n\n    # Check manager Theo Mbeki\n    if 'mbeki' not in fmla_str and 'theo' not in fmla_str:\n        issues.append('FMLA row missing manager Theo Mbeki')\n    if 'mbeki' not in famli_str and 'theo' not in famli_str:\n        issues.append('FAMLI row missing manager Theo Mbeki')\n\n    # Check Foreseeable\n    if 'foreseeable' not in fmla_str:\n        issues.append('FMLA row missing Foreseeable')\n    if 'foreseeable' not in famli_str:\n        issues.append('FAMLI row missing Foreseeable')\n\n    # Check Personal Email\n    if 'personal' not in fmla_str:\n        issues.append('FMLA row missing Personal Email contact method')\n    if 'personal' not in famli_str:\n        issues.append('FAMLI row missing Personal Email contact method')\n\n    # Check Supply Chain\n    if 'supply' not in fmla_str:\n        issues.append('FMLA row missing Supply Chain department')\n    if 'supply' not in famli_str:\n        issues.append('FAMLI row missing Supply Chain department')\n\n    # Check unique Case IDs\n    if case_col is not None:\n        fmla_case = get_val(fmla_row, case_col)\n        famli_case = get_val(famli_row, case_col)\n        if fmla_case and famli_case and fmla_case == famli_case:\n            issues.append(f'FMLA and FAMLI rows have the same Case ID: {fmla_case}')\n        if not fmla_case:\n            issues.append('FMLA row missing Case ID')\n        if not famli_case:\n            issues.append('FAMLI row missing Case ID')\n    else:\n        # Try to extract case IDs from row strings\n        fmla_cases = re.findall(r'[a-z]{2,5}-?\\d{4,}', fmla_str)\n        famli_cases = re.findall(r'[a-z]{2,5}-?\\d{4,}', famli_str)\n        if fmla_cases and famli_cases and fmla_cases[0] == famli_cases[0]:\n            issues.append(f'FMLA and FAMLI rows appear to have the same Case ID')\n\n    if issues:\n        return {'pass': False, 'score': 0.0, 'feedback': 'leave_tracker.xlsx issues: ' + '; '.join(issues)}\n\n    return {'pass': True, 'score': 1.0, 'feedback': f'Found valid FMLA (Denied) and FAMLI (Intake) rows for Portia Nzinga/EMP-1222 with all required fields.'}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "9b36db1e-acac-4dd5-809e-0bc59f961d90",
    "sort_order": 1,
    "rubric_text": "In file `leave_tracker.xlsx`, the FMLA row for Portia Nzinga must contain a note about denial based on insufficient hours, and the FAMLI row must contain a note about template 31 sent to the employee.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n    leave_tracker = None\n    for f in xlsx_files:\n        if 'leave' in f.name.lower() and 'tracker' in f.name.lower():\n            leave_tracker = f\n            break\n    if leave_tracker is None:\n        for f in xlsx_files:\n            if 'leave' in f.name.lower():\n                leave_tracker = f\n                break\n    if leave_tracker is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'No leave_tracker.xlsx file found in workspace'}\n\n    try:\n        wb = openpyxl.load_workbook(leave_tracker)\n        ws = wb.active\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not open leave tracker: {e}'}\n\n    headers = []\n    data_rows = []\n    for i, row in enumerate(ws.iter_rows(values_only=True)):\n        if i == 0:\n            headers = [str(c).strip().lower() if c else '' for c in row]\n        else:\n            if any(c is not None for c in row):\n                data_rows.append(row)\n\n    nzinga_rows = []\n    for row in data_rows:\n        row_str = ' '.join(str(c).lower() for c in row if c is not None)\n        if 'nzinga' in row_str or 'emp-1222' in row_str.replace(' ', '') or '1222' in row_str:\n            nzinga_rows.append(row)\n\n    if len(nzinga_rows) < 2:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Expected at least 2 rows for Portia Nzinga/EMP-1222, found {len(nzinga_rows)}'}\n\n    # Find leave type column for accurate FMLA/FAMLI row detection\n    leave_type_col = None\n    for idx, h in enumerate(headers):\n        if 'leave type' in h or 'leave_type' in h:\n            leave_type_col = idx\n            break\n    if leave_type_col is None:\n        for idx, h in enumerate(headers):\n            if 'type' in h:\n                leave_type_col = idx\n                break\n\n    def get_lt(row, col):\n        if col is None or col >= len(row):\n            return ''\n        v = row[col]\n        return str(v).strip().lower() if v is not None else ''\n\n    fmla_row = None\n    famli_row = None\n    for row in nzinga_rows:\n        row_str = ' '.join(str(c).lower() for c in row if c is not None)\n        lt = get_lt(row, leave_type_col) if leave_type_col is not None else row_str\n        if ('fmla' in lt or 'fmla' in row_str) and 'famli' not in lt:\n            fmla_row = row\n        if 'famli' in lt or 'famli' in row_str:\n            famli_row = row\n\n    if fmla_row is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'Could not find FMLA row for Portia Nzinga'}\n    if famli_row is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'Could not find FAMLI row for Portia Nzinga'}\n\n    fmla_str = ' '.join(str(c).lower() for c in fmla_row if c is not None)\n    famli_str = ' '.join(str(c).lower() for c in famli_row if c is not None)\n\n    issues = []\n\n    # FMLA row should mention denial due to insufficient hours\n    fmla_note_ok = ('hour' in fmla_str and ('insufficient' in fmla_str or 'ineligible' in fmla_str or 'denied' in fmla_str))\n    if not fmla_note_ok:\n        # Also accept if note column contains hours-related denial\n        fmla_note_ok = 'hour' in fmla_str\n    if not fmla_note_ok:\n        issues.append(f'FMLA row missing note about denial based on insufficient hours. Row content: {fmla_str[:200]}')\n\n    # FAMLI row should mention template 31\n    famli_note_ok = ('template 31' in famli_str or 'template31' in famli_str or ('template' in famli_str and '31' in famli_str) or 'cdle' in famli_str or 'claims@famli' in famli_str or ('famli' in famli_str and 'acknowledg' in famli_str))\n    if not famli_note_ok:\n        issues.append(f'FAMLI row missing note about template 31. Row content: {famli_str[:200]}')\n\n    if issues:\n        return {'pass': False, 'score': 0.0, 'feedback': 'Notes issues: ' + '; '.join(issues)}\n\n    return {'pass': True, 'score': 1.0, 'feedback': 'FMLA row has denial/insufficient hours note and FAMLI row has template 31 note.'}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "63872e44-3fe8-4441-8b82-e77d0bc15ba2",
    "sort_order": 2,
    "rubric_text": "In `leave_tracker.xlsx`, the FMLA and FAMLI rows for Portia Nzinga must have two different (unique) Case IDs.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n    leave_tracker = None\n    for f in xlsx_files:\n        if 'leave' in f.name.lower() and 'tracker' in f.name.lower():\n            leave_tracker = f\n            break\n    if leave_tracker is None:\n        for f in xlsx_files:\n            if 'leave' in f.name.lower():\n                leave_tracker = f\n                break\n    if leave_tracker is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'No leave_tracker.xlsx file found in workspace'}\n\n    try:\n        wb = openpyxl.load_workbook(leave_tracker)\n        ws = wb.active\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not open leave tracker: {e}'}\n\n    headers = []\n    data_rows = []\n    for i, row in enumerate(ws.iter_rows(values_only=True)):\n        if i == 0:\n            headers = [str(c).strip().lower() if c else '' for c in row]\n        else:\n            if any(c is not None for c in row):\n                data_rows.append(row)\n\n    # Find case ID column\n    case_col = None\n    for idx, h in enumerate(headers):\n        if 'case' in h and ('id' in h or 'num' in h or 'no' in h):\n            case_col = idx\n            break\n    if case_col is None:\n        for idx, h in enumerate(headers):\n            if 'case' in h:\n                case_col = idx\n                break\n\n    nzinga_rows = []\n    for row in data_rows:\n        row_str = ' '.join(str(c).lower() for c in row if c is not None)\n        if 'nzinga' in row_str or 'emp-1222' in row_str.replace(' ', '') or '1222' in row_str:\n            nzinga_rows.append(row)\n\n    if len(nzinga_rows) < 2:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Expected at least 2 rows for Portia Nzinga, found {len(nzinga_rows)}'}\n\n    # Find leave type column for accurate FMLA/FAMLI row detection\n    leave_type_col = None\n    for idx, h in enumerate(headers):\n        if 'leave type' in h or 'leave_type' in h:\n            leave_type_col = idx\n            break\n    if leave_type_col is None:\n        for idx, h in enumerate(headers):\n            if 'type' in h:\n                leave_type_col = idx\n                break\n\n    def get_lt(row, col):\n        if col is None or col >= len(row):\n            return ''\n        v = row[col]\n        return str(v).strip().lower() if v is not None else ''\n\n    fmla_row = None\n    famli_row = None\n    for row in nzinga_rows:\n        row_str = ' '.join(str(c).lower() for c in row if c is not None)\n        lt = get_lt(row, leave_type_col) if leave_type_col is not None else row_str\n        if ('fmla' in lt or 'fmla' in row_str) and 'famli' not in lt:\n            fmla_row = row\n        if 'famli' in lt or 'famli' in row_str:\n            famli_row = row\n\n    if fmla_row is None or famli_row is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'Could not identify both FMLA and FAMLI rows for Portia Nzinga'}\n\n    if case_col is not None:\n        fmla_case = str(fmla_row[case_col]).strip() if case_col < len(fmla_row) and fmla_row[case_col] is not None else ''\n        famli_case = str(famli_row[case_col]).strip() if case_col < len(famli_row) and famli_row[case_col] is not None else ''\n    else:\n        # Try to extract case IDs from row strings\n        fmla_str = ' '.join(str(c) for c in fmla_row if c is not None)\n        famli_str = ' '.join(str(c) for c in famli_row if c is not None)\n        fmla_cases = re.findall(r'[A-Za-z]{2,5}-?\\d{4,}', fmla_str)\n        famli_cases = re.findall(r'[A-Za-z]{2,5}-?\\d{4,}', famli_str)\n        fmla_case = fmla_cases[0] if fmla_cases else ''\n        famli_case = famli_cases[0] if famli_cases else ''\n\n    if not fmla_case:\n        return {'pass': False, 'score': 0.0, 'feedback': 'FMLA row has no Case ID'}\n    if not famli_case:\n        return {'pass': False, 'score': 0.0, 'feedback': 'FAMLI row has no Case ID'}\n    if fmla_case.lower() == famli_case.lower():\n        return {'pass': False, 'score': 0.0, 'feedback': f'FMLA and FAMLI rows share the same Case ID: {fmla_case}'}\n\n    return {'pass': True, 'score': 1.0, 'feedback': f'FMLA Case ID ({fmla_case}) and FAMLI Case ID ({famli_case}) are different and unique.'}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "003231a0-0a79-4c09-8365-3fb47ed32a32",
    "sort_order": 3,
    "rubric_text": "In `famli_coordination_log.xlsx`, no new rows for Portia Nzinga or EMP-1222 must have been added.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n    famli_log = None\n    for f in xlsx_files:\n        if 'famli' in f.name.lower() and ('coord' in f.name.lower() or 'log' in f.name.lower()):\n            famli_log = f\n            break\n    if famli_log is None:\n        for f in xlsx_files:\n            if 'famli' in f.name.lower():\n                famli_log = f\n                break\n    if famli_log is None:\n        # If file doesn't exist, no rows were added - pass\n        return {'pass': True, 'score': 1.0, 'feedback': 'No famli_coordination_log.xlsx found; no new rows added (file absent is acceptable).'}\n\n    try:\n        wb = openpyxl.load_workbook(famli_log)\n        ws = wb.active\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not open famli coordination log: {e}'}\n\n    for row in ws.iter_rows(values_only=True):\n        row_str = ' '.join(str(c).lower() for c in row if c is not None)\n        if 'nzinga' in row_str or '1222' in row_str:\n            return {'pass': False, 'score': 0.0, 'feedback': f'Found a row for Portia Nzinga/EMP-1222 in famli_coordination_log.xlsx, but no new rows should have been added. Row: {row}'}\n\n    return {'pass': True, 'score': 1.0, 'feedback': 'No rows for Portia Nzinga/EMP-1222 found in famli_coordination_log.xlsx.'}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "46a3c331-553d-4857-b1f8-79c63468721e",
    "sort_order": 4,
    "rubric_text": "In mailbox.json, a sent email to p.nzinga@ridgelinegear.com must exist that uses template 3 (FMLA ineligibility/denial notification) with CC to r.okafor@ridgelinegear.com and has a WH-381 attachment.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n    try:\n        with open(mailbox_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n    emails = data.get('emails', [])\n    # Identify outbound emails to Nzinga by from_addr (not folder, since the agent may move emails to case folders).\n    # An email is 'sent' if it was authored by someone other than Nzinga and addressed to Nzinga.\n    sent_to_nzinga = [\n        e for e in emails\n        if 'p.nzinga@ridgelinegear.com' in (e.get('to_addr', '') or '')\n        and 'p.nzinga@ridgelinegear.com' not in (e.get('from_addr', '') or '').lower()\n        and (e.get('from_addr', '') or '').strip() != ''\n    ]\n\n    if not sent_to_nzinga:\n        return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails to p.nzinga@ridgelinegear.com found'}\n\n    # Look for the template 3 email (ineligibility/denial) with WH-381 and CC to r.okafor\n    found = False\n    for e in sent_to_nzinga:\n        body = (e.get('body_text', '') or '').lower()\n        subject = (e.get('subject', '') or '').lower()\n        cc = (e.get('cc_addr', '') or '').lower()\n        attachments = e.get('attachments', []) or []\n        att_names = ' '.join(str(a).lower() for a in attachments)\n\n        has_cc_okafor = 'r.okafor@ridgelinegear.com' in cc\n        has_wh381 = 'wh-381' in att_names or 'wh381' in att_names or 'wh_381' in att_names\n        # Template 3 likely references ineligibility/denial\n        has_template3_content = (\n            'template 3' in body or 'template3' in body or\n            'inelig' in body or 'denied' in body or 'not eligible' in body or\n            'inelig' in subject or 'denied' in subject or\n            'wh-381' in body or 'wh381' in body\n        )\n\n        if has_cc_okafor and has_wh381:\n            found = True\n            break\n        # More lenient: cc + template 3 content\n        if has_cc_okafor and has_template3_content:\n            found = True\n            break\n\n    if not found:\n        details = [f\"Email {i}: CC={e.get('cc_addr')}, attachments={e.get('attachments')}, subject={e.get('subject')}\" for i, e in enumerate(sent_to_nzinga)]\n        return {'pass': False, 'score': 0.0, 'feedback': f'No sent email to p.nzinga@ridgelinegear.com with CC to r.okafor@ridgelinegear.com and WH-381 attachment found. Emails found: {details}'}\n\n    return {'pass': True, 'score': 1.0, 'feedback': 'Found sent email to p.nzinga@ridgelinegear.com with CC to r.okafor@ridgelinegear.com and WH-381 attachment.'}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "d32a0eda-68e8-4ebf-8c59-3d7dd94b6bbf",
    "sort_order": 5,
    "rubric_text": "In mailbox.json, a sent email to p.nzinga@ridgelinegear.com must exist using template 31 with no CC.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n    try:\n        with open(mailbox_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n    emails = data.get('emails', [])\n    sent_to_nzinga = [\n        e for e in emails\n        if e.get('folder', '').lower() == 'sent'\n        and 'p.nzinga@ridgelinegear.com' in (e.get('to_addr', '') or '')\n    ]\n\n    if not sent_to_nzinga:\n        return {'pass': False, 'score': 0.0, 'feedback': 'No sent emails to p.nzinga@ridgelinegear.com found'}\n\n    for e in sent_to_nzinga:\n        body = (e.get('body_text', '') or '').lower()\n        subject = (e.get('subject', '') or '').lower()\n        cc = (e.get('cc_addr', '') or '').strip()\n\n        has_template31 = (\n            ('famli' in body and ('cdle' in body or 'claims@famli' in body))\n            or 'leave request received' in subject\n        )\n        no_cc = not cc or cc.lower() == 'none'\n\n        if has_template31 and no_cc:\n            return {'pass': True, 'score': 1.0, 'feedback': 'Found sent email to p.nzinga@ridgelinegear.com using template 31 with no CC.'}\n\n    details = [\n        f\"Email {i}: CC={e.get('cc_addr')}, subject={e.get('subject')}, \"\n        f\"has_template31={'template 31' in (e.get('body_text','') or '').lower() or 'template31' in (e.get('body_text','') or '').lower()}\"\n        for i, e in enumerate(sent_to_nzinga)\n    ]\n    return {\n        'pass': False,\n        'score': 0.0,\n        'feedback': f'No sent email to p.nzinga@ridgelinegear.com found that explicitly references template 31 with no CC. Emails found: {details}'\n    }",
    "criterion_type": "expected_output"
  },
  {
    "id": "2dfcfc8e-f3c1-450e-9d35-dd8d3248af40",
    "sort_order": 6,
    "rubric_text": "In mailbox.json, the original email from p.nzinga@ridgelinegear.com must be marked as read and moved to a 'FAMLI Cases' folder.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n    try:\n        with open(mailbox_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n    emails = data.get('emails', [])\n\n    # Find the original email FROM p.nzinga@ridgelinegear.com\n    nzinga_emails = [\n        e for e in emails\n        if 'p.nzinga@ridgelinegear.com' in (e.get('from_addr', '') or '')\n    ]\n\n    if not nzinga_emails:\n        return {'pass': False, 'score': 0.0, 'feedback': 'No emails from p.nzinga@ridgelinegear.com found'}\n\n    issues = []\n    found_in_famli_folder = False\n    found_marked_read = False\n\n    for e in nzinga_emails:\n        folder = (e.get('folder', '') or '').lower().replace(' ', '').replace('_', '')\n        is_read = e.get('is_read', False)\n\n        if 'famli' in folder and 'case' in folder:\n            found_in_famli_folder = True\n            if is_read:\n                found_marked_read = True\n\n    if not found_in_famli_folder:\n        folders = [e.get('folder') for e in nzinga_emails]\n        issues.append(f\"Email from p.nzinga not in 'FAMLI Cases' folder. Found folders: {folders}\")\n    if not found_marked_read:\n        read_statuses = [e.get('is_read') for e in nzinga_emails]\n        issues.append(f\"Email from p.nzinga not marked as read. Read statuses: {read_statuses}\")\n\n    if issues:\n        return {'pass': False, 'score': 0.0, 'feedback': '; '.join(issues)}\n\n    return {'pass': True, 'score': 1.0, 'feedback': \"Email from p.nzinga@ridgelinegear.com is marked as read and located in 'FAMLI Cases' folder.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "cf793b87-1f07-40a7-8d80-97c250d2db95",
    "sort_order": 7,
    "rubric_text": "In mailbox.json, no sent email to the manager (Theo Mbeki) must exist related to Portia Nzinga's leave request.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check mailbox'}\n\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n\n    try:\n        with open(mailbox_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n    emails = data.get('emails', [])\n\n    contacts = data.get('contacts', [])\n    mbeki_email = None\n    for c in contacts:\n        name = (c.get('name', '') or '').lower()\n        if 'mbeki' in name:\n            mbeki_email = (c.get('email', '') or '').lower()\n            break\n\n    nzinga_identifiers = {'nzinga', '1222', 'emp-1222'}\n\n    sent_to_mbeki_about_nzinga = []\n    for e in emails:\n        if e.get('folder', '').lower() != 'sent':\n            continue\n\n        to_addr = (e.get('to_addr', '') or '').lower()\n        cc_addr = (e.get('cc_addr', '') or '').lower()\n\n        addressed_to_mbeki = (\n            (mbeki_email and (mbeki_email in to_addr or mbeki_email in cc_addr)) or\n            't.mbeki@ridgelinegear.com' in to_addr or\n            't.mbeki@ridgelinegear.com' in cc_addr or\n            't.mbeki' in to_addr or\n            't.mbeki' in cc_addr\n        )\n        if not addressed_to_mbeki:\n            continue\n\n        body = (e.get('body_text', '') or '').lower()\n        subject = (e.get('subject', '') or '').lower()\n        combined = body + ' ' + subject\n\n        if any(identifier in combined for identifier in nzinga_identifiers):\n            sent_to_mbeki_about_nzinga.append(e)\n\n    if sent_to_mbeki_about_nzinga:\n        details = [(e.get('subject'), e.get('to_addr')) for e in sent_to_mbeki_about_nzinga]\n        return {\n            'pass': False,\n            'score': 0.0,\n            'feedback': (\n                f'Found {len(sent_to_mbeki_about_nzinga)} sent email(s) to manager Theo Mbeki '\n                f'specifically about Portia Nzinga leave, but none should have been sent. '\n                f'Emails: {details}'\n            )\n        }\n\n    return {\n        'pass': True,\n        'score': 1.0,\n        'feedback': 'No sent emails to manager Theo Mbeki about Portia Nzinga leave found (as expected).'\n    }\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "1d87f034-4cd2-4b70-8022-5be47f02af10",
    "sort_order": 8,
    "rubric_text": "A WH-381 form with Part A filled out must exist in the workspace, marked 'You are not eligible'.",
    "verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    # Check for WH-381 in workspace files\n    workspace = Path(workspace_path)\n\n    # Search for WH-381 file with various extensions\n    wh381_files = []\n    for pattern in ['*WH-381*', '*WH381*', '*wh-381*', '*wh381*', '*wh_381*']:\n        wh381_files.extend(workspace.glob(f'**/{pattern}'))\n\n    # Also check attachments or email-related folders\n    if not wh381_files:\n        # Check if it might be referenced in any file\n        all_files = list(workspace.glob('**/*'))\n        for f in all_files:\n            if 'wh' in f.name.lower() and ('381' in f.name):\n                wh381_files.append(f)\n\n    # Check external services for the attachment\n    if not wh381_files and external_services_path:\n        mailbox_path = Path(external_services_path) / 'mailbox.json'\n        if mailbox_path.exists():\n            import json\n            try:\n                with open(mailbox_path) as f:\n                    data = json.load(f)\n                emails = data.get('emails', [])\n                for e in emails:\n                    if e.get('folder', '').lower() == 'sent':\n                        attachments = e.get('attachments', []) or []\n                        for att in attachments:\n                            att_str = str(att).lower()\n                            if 'wh-381' in att_str or 'wh381' in att_str or 'wh_381' in att_str:\n                                # Found as an attachment in a sent email\n                                # Check the attachment content if available\n                                body = (e.get('body_text', '') or '').lower()\n                                has_part_a = 'part a' in body or 'parta' in body\n                                has_ineligible = 'not eligible' in body or 'ineligible' in body\n                                if has_part_a or has_ineligible:\n                                    return {'pass': True, 'score': 1.0, 'feedback': f'WH-381 found as attachment in sent email with Part A / ineligibility indicators.'}\n                                # If attachment exists in sent email, that's likely sufficient\n                                return {'pass': True, 'score': 1.0, 'feedback': f'WH-381 found as attachment in sent email.'}\n            except Exception:\n                pass\n\n    if not wh381_files:\n        # Final check: look in any PDF or docx files for WH-381 content\n        found_content = False\n        for ext in ['*.pdf', '*.docx', '*.doc']:\n            for f in workspace.glob(f'**/{ext}'):\n                fname = f.name.lower()\n                if 'wh' in fname or '381' in fname:\n                    wh381_files.append(f)\n                    found_content = True\n\n        if not wh381_files:\n            return {'pass': False, 'score': 0.0, 'feedback': 'No WH-381 form file found in workspace or mailbox attachments'}\n\n    # Try to read content from found files\n    for f in wh381_files:\n        content = ''\n        try:\n            if f.suffix.lower() == '.pdf':\n                import pdfplumber\n                with pdfplumber.open(f) as pdf:\n                    content = ' '.join(page.extract_text() or '' for page in pdf.pages).lower()\n            elif f.suffix.lower() in ['.docx']:\n                from docx import Document\n                doc = Document(f)\n                content = ' '.join(p.text for p in doc.paragraphs).lower()\n            elif f.suffix.lower() in ['.txt', '.html', '.htm']:\n                content = f.read_text(errors='ignore').lower()\n            else:\n                content = f.name.lower()  # At minimum check filename\n        except Exception:\n            content = f.name.lower()\n\n        has_part_a = 'part a' in content or 'section 1' in content or 'employer' in content\n        has_ineligible = 'not eligible' in content or 'ineligible' in content or 'you are not eligible' in content\n\n        if has_part_a and has_ineligible:\n            return {'pass': True, 'score': 1.0, 'feedback': f'WH-381 found at {f} with Part A filled out and \"not eligible\" indication.'}\n\n        # If we found the file but couldn't verify content, give partial pass\n        if 'wh-381' in f.name.lower() or 'wh381' in f.name.lower():\n            return {'pass': True, 'score': 0.8, 'feedback': f'WH-381 file found at {f}; could not fully verify Part A and ineligibility marking from content.'}\n\n    return {'pass': False, 'score': 0.0, 'feedback': f'WH-381 files found ({wh381_files}) but could not verify Part A filled out and \"not eligible\" marking.'}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "f8ed7e1c-d124-45a1-92ee-a0ee9aea0672",
    "sort_order": 9,
    "rubric_text": "In slack_data.json, a message must exist in #leave-intake channel containing 'FMLA DENIED (INELIGIBILITY)' followed by a Case ID matching the FMLA Case ID in leave_tracker.xlsx and a reference to hours.",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check Slack'}\n\n    slack_path = Path(external_services_path) / 'slack_data.json'\n    if not slack_path.exists():\n        return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found'}\n\n    try:\n        with open(slack_path) as f:\n            slack_data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse slack_data.json: {e}'}\n\n    # Get FMLA Case ID from leave_tracker\n    fmla_case_id = None\n    xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n    leave_tracker = None\n    for f in xlsx_files:\n        if 'leave' in f.name.lower() and 'tracker' in f.name.lower():\n            leave_tracker = f\n            break\n    if leave_tracker is None:\n        for f in xlsx_files:\n            if 'leave' in f.name.lower():\n                leave_tracker = f\n                break\n\n    if leave_tracker:\n        try:\n            wb = openpyxl.load_workbook(leave_tracker)\n            ws = wb.active\n            headers = []\n            for i, row in enumerate(ws.iter_rows(values_only=True)):\n                if i == 0:\n                    headers = [str(c).strip().lower() if c else '' for c in row]\n                    break\n            case_col = None\n            for idx, h in enumerate(headers):\n                if 'case' in h and ('id' in h or 'num' in h or 'no' in h):\n                    case_col = idx\n                    break\n            if case_col is None:\n                for idx, h in enumerate(headers):\n                    if 'case' in h:\n                        case_col = idx\n                        break\n            leave_type_col = None\n            for idx, h in enumerate(headers):\n                if 'leave type' in h or 'leave_type' in h:\n                    leave_type_col = idx\n                    break\n            if leave_type_col is None:\n                for idx, h in enumerate(headers):\n                    if 'type' in h:\n                        leave_type_col = idx\n                        break\n\n            for i, row in enumerate(ws.iter_rows(values_only=True)):\n                if i == 0:\n                    continue\n                row_str = ' '.join(str(c).lower() for c in row if c is not None)\n                lt = str(row[leave_type_col]).strip().lower() if (leave_type_col is not None and leave_type_col < len(row) and row[leave_type_col] is not None) else ''\n                if ('nzinga' in row_str or '1222' in row_str) and 'fmla' in row_str and 'famli' not in lt:\n                    if case_col is not None and case_col < len(row) and row[case_col]:\n                        fmla_case_id = str(row[case_col]).strip()\n                    else:\n                        # Try to extract from row\n                        cases = re.findall(r'[A-Za-z]{2,5}-?\\d{4,}', ' '.join(str(c) for c in row if c is not None))\n                        if cases:\n                            fmla_case_id = cases[0]\n                    break\n        except Exception:\n            pass\n\n    # Find #leave-intake channel\n    channels = slack_data.get('channels', {})\n    leave_intake_channel_id = None\n    for ch_id, ch_data in channels.items():\n        if 'leave' in ch_data.get('name', '').lower() and 'intake' in ch_data.get('name', '').lower():\n            leave_intake_channel_id = ch_id\n            break\n    if leave_intake_channel_id is None:\n        for ch_id, ch_data in channels.items():\n            if 'leave' in ch_data.get('name', '').lower():\n                leave_intake_channel_id = ch_id\n                break\n\n    if leave_intake_channel_id is None:\n        return {'pass': False, 'score': 0.0, 'feedback': '#leave-intake channel not found in Slack'}\n\n    messages = slack_data.get('messages', {})\n    channel_messages = messages.get(leave_intake_channel_id, [])\n\n    found = False\n    for msg in channel_messages:\n        text = (msg.get('text', '') or '').upper()\n        if 'FMLA DENIED (INELIGIBILITY)' in text or 'FMLA DENIED' in text:\n            # Check for case ID match\n            has_case_id = True  # Default True if we couldn't get case ID\n            if fmla_case_id:\n                has_case_id = fmla_case_id.upper() in text\n            # Check for hours reference\n            text_lower = msg.get('text', '').lower()\n            has_hours = 'hour' in text_lower\n            if has_case_id and has_hours:\n                found = True\n                break\n            elif has_case_id:\n                # Slightly lenient: if we found case ID, accept even without explicit 'hours' word\n                found = True\n                break\n\n    if not found:\n        all_texts = [msg.get('text', '')[:100] for msg in channel_messages]\n        return {'pass': False, 'score': 0.0, 'feedback': f'No Slack message in #leave-intake with FMLA DENIED (INELIGIBILITY) + Case ID ({fmla_case_id}) + hours found. Messages: {all_texts}'}\n\n    return {'pass': True, 'score': 1.0, 'feedback': f'Found Slack message in #leave-intake with FMLA DENIED (INELIGIBILITY) and Case ID ({fmla_case_id}) and hours reference.'}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "9444eb3e-8313-4c0e-9a60-b4910a9fb610",
    "sort_order": 10,
    "rubric_text": "In calendar_data.json, no new calendar events related to Portia Nzinga's leave (EMP-1222) must have been added.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        # No calendar service to check - pass by default\n        return {'pass': True, 'score': 1.0, 'feedback': 'external_services_path not provided; no calendar entries to check.'}\n\n    calendar_path = Path(external_services_path) / 'calendar_data.json'\n    if not calendar_path.exists():\n        return {'pass': True, 'score': 1.0, 'feedback': 'calendar_data.json not found; no new events added (as expected).'}\n\n    try:\n        with open(calendar_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse calendar_data.json: {e}'}\n\n    events = data.get('events', {})\n    nzinga_events = []\n    for event_id, event in events.items():\n        summary = (event.get('summary', '') or '').lower()\n        if 'nzinga' in summary or '1222' in summary or ('portia' in summary and 'leave' in summary):\n            nzinga_events.append((event_id, event))\n\n    if nzinga_events:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Found {len(nzinga_events)} calendar event(s) related to Portia Nzinga, but no events should have been added. Events: {nzinga_events}'}\n\n    return {'pass': True, 'score': 1.0, 'feedback': 'No calendar events related to Portia Nzinga found (as expected).'}\n",
    "criterion_type": "incorrect_behavior"
  }
]