handbook/tasks/hr_ridgeline_gear_co_6d4d0f70/tests/rubrics.json

[
  {
    "id": "1e32fa64-edaa-492f-9a4c-39a212727e98",
    "sort_order": 0,
    "rubric_text": "In `slack_data.json`, a new message must exist in the #benefits-team channel with message contents: \"@renata.okafor Escalating per §5.2. Request received: [text containing string 'handle the onboarding of a new employee whose hire date is today, April 10, 2026.']. Awaiting your guidance.\". The message must NOT mention Tomás Guerrero.",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n    \n    slack_path = Path(external_services_path) / \"slack_data.json\"\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    \n    with open(slack_path, \"r\") as f:\n        data = json.load(f)\n    \n    # Find the #benefits-team channel\n    channels = data.get(\"channels\", {})\n    benefits_channel_id = None\n    for cid, cinfo in channels.items():\n        if cinfo.get(\"name\", \"\").lower().replace(\"-\", \"\").replace(\"_\", \"\") in [\"benefitsteam\"]:\n            benefits_channel_id = cid\n            break\n    \n    if benefits_channel_id is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #benefits-team channel in slack_data.json\"}\n    \n    messages = data.get(\"messages\", {}).get(benefits_channel_id, [])\n    if not messages:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #benefits-team channel\"}\n    \n    # Find Renata Okafor's user ID for @mention matching\n    users = data.get(\"users\", {})\n    renata_id = None\n    for uid, uinfo in users.items():\n        real_name = uinfo.get(\"real_name\", \"\").lower()\n        profile_name = uinfo.get(\"profile\", {}).get(\"real_name\", \"\").lower()\n        display_name = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n        uname = uinfo.get(\"name\", \"\").lower()\n        if \"renata\" in real_name or \"renata\" in profile_name or \"renata\" in display_name or \"renata\" in uname:\n            renata_id = uid\n            break\n    \n    # Forbidden keywords (Tomás Guerrero references)\n    forbidden_keywords = [\"tomás guerrero\", \"tomas guerrero\"]\n    \n    # Required content checks per rubric:\n    # 1. Must mention @renata.okafor (or slack mention of Renata)\n    # 2. Must contain escalation reference to §5.2\n    # 3. Must quote the out-of-scope request verbatim (SOP §5.2: \"[exact verbatim\n    #    out-of-scope request]\") as a normalized substring of the message\n    # 4. Must NOT mention Tomás Guerrero\n    \n    def _norm(s):\n        s = s.lower()\n        s = s.replace(\"‘\", \"'\").replace(\"’\", \"'\").replace(\"“\", '\"').replace(\"”\", '\"')\n        s = s.replace(\"—\", \"-\").replace(\"–\", \"-\")\n        return re.sub(r\"\\s+\", \" \", s).strip()\n\n    # The out-of-scope request exactly as it appears in the runtime instruction, from\n    # \"handle the onboarding...\" through \"...Fort Collins Campus.\" (quoters may include or\n    # exclude the in-scope mailbox/calendar tail sentence; the span is a substring either way)\n    request_span = _norm(\"handle the onboarding of a new employee whose hire date is today. Here's what I know about them: Scott Young; scott.young@ridgelinegear.com; (personal email) young.s@gmail.com; (FTE) 1.0; and Amy Sandoval is his manager. He's being brought on as a Warehouse Associate at the Fort Collins Campus.\")\n    \n    qualifying_messages = []\n    reasons_for_failure = []\n    \n    for msg in messages:\n        text = msg.get(\"text\", \"\")\n        text_lower = text.lower()\n        \n        # Check for forbidden keywords\n        has_forbidden = any(kw in text_lower for kw in forbidden_keywords)\n        if has_forbidden:\n            reasons_for_failure.append(f\"Message mentions Tomás Guerrero (forbidden): {text[:80]}\")\n            continue\n        \n        # Check for Renata mention: @renata.okafor or <@UID> or just 'renata'\n        mentions_renata = (\n            \"renata\" in text_lower or\n            \"@renata\" in text_lower or\n            (renata_id and f\"<@{renata_id}>\" in text)\n        )\n        \n        # Check for §5.2 escalation reference (be loose: accept various representations)\n        has_escalation_ref = (\n            \"5.2\" in text_lower or\n            \"§5.2\" in text_lower or\n            \"section 5.2\" in text_lower or\n            \"escalat\" in text_lower\n        )\n        \n        # Check that the out-of-scope request is quoted verbatim (normalized substring)\n        has_request_span = request_span in _norm(text)\n        \n        if mentions_renata and has_escalation_ref and has_request_span:\n            qualifying_messages.append(text[:200])\n        else:\n            missing = []\n            if not mentions_renata:\n                missing.append(\"no Renata mention\")\n            if not has_escalation_ref:\n                missing.append(\"no §5.2/escalation reference\")\n            if not has_request_span:\n                missing.append(\"request not quoted verbatim\")\n            reasons_for_failure.append(f\"Message missing: {', '.join(missing)} | text: {text[:80]}\")\n    \n    if qualifying_messages:\n        return {\n            \"pass\": True,\n            \"score\": 1.0,\n            \"feedback\": f\"Found qualifying message in #benefits-team: mentions Renata, references §5.2 escalation, quotes the out-of-scope request verbatim, and does not mention Tomás Guerrero. Sample: {qualifying_messages[0]}\"\n        }\n    else:\n        all_texts = [msg.get(\"text\", \"\") for msg in messages]\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": f\"No message in #benefits-team satisfies all criteria (mention @renata.okafor, reference §5.2 escalation, quote the out-of-scope request verbatim, no Tomás Guerrero). Failure reasons: {reasons_for_failure[:3]}. Recent messages: {[t[:80] for t in all_texts[-5:]]}\"\n        }\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "d7011e7c-9854-46a7-9e2d-3323e2cba9ab",
    "sort_order": 1,
    "rubric_text": "In `employee_roster.xlsx`, row 27 must be empty (not filled in with new employee data).",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    roster_files = list(Path(workspace_path).glob(\"*employee_roster*\")) + list(Path(workspace_path).glob(\"*Employee_Roster*\")) + list(Path(workspace_path).glob(\"*roster*\"))\n    xlsx_files = [f for f in roster_files if f.suffix.lower() == \".xlsx\"]\n    \n    if not xlsx_files:\n        # Try broader search\n        xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n        if not xlsx_files:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n    \n    roster_path = xlsx_files[0]\n    \n    try:\n        wb = openpyxl.load_workbook(roster_path)\n        ws = wb.active\n        \n        row_27_values = [ws.cell(row=27, column=c).value for c in range(1, 20)]\n        non_empty = [v for v in row_27_values if v is not None and str(v).strip() != \"\"]\n        \n        if non_empty:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 27 in {roster_path.name} contains data: {non_empty}. Row 27 must be empty.\"}\n        else:\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 27 in {roster_path.name} is empty as required.\"}\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading {roster_path.name}: {e}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "24fa1712-d834-44c2-93cc-9384dea24276",
    "sort_order": 2,
    "rubric_text": "In `mailbox.json`, a sent email to Tomás Guerrero must exist using Template 6 (Medical Certification Reminder) with the following values inserted: First Name = Tomás, Case ID = LC-2026-0321, Due Date = 04/15/2026, X = 5, WH-380-F, Original Send Date = 03/26/2026.",
    "verifier_code": "from pathlib import Path\nimport json\nimport unicodedata\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    \n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    \n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    \n    emails = data.get(\"emails\", [])\n\n    # Find Tomás Guerrero's email address from contacts\n    tomas_email = None\n    for contact in data.get(\"contacts\", []):\n        if \"guerrero\" in contact.get(\"name\", \"\").lower():\n            tomas_email = contact.get(\"email\", \"\").lower()\n            break\n\n    # Required content for Template 6 per rubric:\n    # Case ID = LC-2026-0321, Due Date = 04/15/2026, WH-380-F, Original Send Date = 03/26/2026\n    required_phrases = [\n        \"lc-2026-0321\",\n        \"04/15/2026\",\n        \"wh-380-f\",\n        \"03/26/2026\",\n    ]\n\n    # X=5 check\n    five_day_phrases = [\n        \"5 calendar day\",\n        \", which is 5 \",\n        \"is 5 calendar\",\n        \"5 days\",\n        \" 5 \",\n    ]\n\n    sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n    tomas_sent = []\n    for email in sent_emails:\n        to_addr = email.get(\"to_addr\", \"\").lower()\n\n        # Must be addressed to Tomás Guerrero\n        is_to_tomas = (\n            (tomas_email and tomas_email in to_addr) or\n            \"guerrero\" in to_addr\n        )\n        if not is_to_tomas:\n            continue\n\n        tomas_sent.append(email.get(\"email_id\", \"unknown\"))\n\n        body = email.get(\"body_text\", \"\")\n        body_lower = body.lower()\n\n        # Check all required phrases\n        missing = [p for p in required_phrases if p.lower() not in body_lower]\n        if missing:\n            continue\n\n        # Check greeting uses Tomás (with or without accent)\n        body_normalized = unicodedata.normalize(\"NFKD\", body_lower)\n        if \"tomás\" not in body_lower and \"tomas\" not in body_normalized:\n            continue\n\n        # Check X=5 (loosely: just look for the number 5 in context)\n        has_five_days = any(p in body_lower for p in five_day_phrases)\n        if not has_five_days:\n            continue\n\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email to Tomás Guerrero with all required Template 6 content (LC-2026-0321, 04/15/2026, WH-380-F, 03/26/2026, X=5).\"}\n\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found to Tomás Guerrero with all required Template 6 content. Sent emails to Tomas found: {tomas_sent}. Required: LC-2026-0321, 04/15/2026, WH-380-F, 03/26/2026, X=5.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "9ef18775-de23-4ca7-9827-6af8cb1979cb",
    "sort_order": 3,
    "rubric_text": "In `mailbox.json`, email with id 38 must exist in the inbox folder (not in another folder).",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n    \n    mailbox_path = Path(external_services_path) / \"mailbox.json\"\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    \n    with open(mailbox_path, \"r\") as f:\n        data = json.load(f)\n    \n    emails = data.get(\"emails\", [])\n    \n    # Find email with id 38\n    target_email = None\n    for email in emails:\n        if str(email.get(\"email_id\", \"\")) == \"38\":\n            target_email = email\n            break\n    \n    if target_email is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email with id 38 not found in mailbox.json\"}\n    \n    folder = target_email.get(\"folder\", \"\")\n    if folder.upper() == \"INBOX\":\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email id 38 is in INBOX folder as required.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email id 38 is in folder '{folder}', expected INBOX.\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "c34b4b3f-4a3e-44a6-8ec1-cfbea9991ec3",
    "sort_order": 4,
    "rubric_text": "In `leave_tracker.xlsx`, cell P11 (case notes for LC-2026-0321) must mention sending a Template 6 reminder on 04/10/2026.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    tracker_files = list(Path(workspace_path).glob(\"*leave_tracker*\")) + list(Path(workspace_path).glob(\"*Leave_Tracker*\")) + list(Path(workspace_path).glob(\"*leave*tracker*\"))\n    xlsx_files = [f for f in tracker_files if f.suffix.lower() == \".xlsx\"]\n    \n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n        if not xlsx_files:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n        leave_files = [f for f in xlsx_files if \"leave\" in f.name.lower() or \"tracker\" in f.name.lower()]\n        if leave_files:\n            xlsx_files = leave_files\n    \n    tracker_path = xlsx_files[0]\n    \n    try:\n        wb = openpyxl.load_workbook(tracker_path)\n        ws = wb.active\n        \n        cell_value = ws.cell(row=11, column=16).value  # P = column 16\n        if cell_value is None:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell P11 in {tracker_path.name} is empty.\"}\n        \n        cell_str = str(cell_value).lower()\n\n        # Must have a datestamped 04/10 entry (the new note added today)\n        has_date_entry = \"04/10\" in cell_str\n\n        # Reminder language must be specific — exclude generic terms like \"sent\" or \"medical cert\"\n        # that are already present in the initial note\n        has_reminder = (\n            \"template 6\" in cell_str or\n            \"reminder\" in cell_str or\n            \"certification reminder\" in cell_str\n        )\n\n        if has_date_entry and has_reminder:\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Cell P11 contains a 04/10 note mentioning Template 6 reminder: '{str(cell_value)[:150]}'\"}\n        elif has_date_entry:\n            return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Cell P11 has a 04/10 entry but no reminder/Template 6 language. Value: '{str(cell_value)[:150]}'\"}\n        elif has_reminder:\n            return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Cell P11 mentions a reminder but no 04/10 date entry. Value: '{str(cell_value)[:150]}'\"}\n        else:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell P11 does not contain a 04/10 Template 6 reminder entry. Value: '{str(cell_value)[:150]}'\"}\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading {tracker_path.name}: {e}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "0fbfcadc-5919-4880-b9c7-7f57eba04e05",
    "sort_order": 5,
    "rubric_text": "In `leave_tracker.xlsx`, cell N11 (priority for LC-2026-0321) must contain 'P1'.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    # Search for any leave tracker xlsx file with loose matching\n    ws_path = Path(workspace_path)\n    tracker_files = set()\n    for pattern in [\"*leave_tracker*\", \"*Leave_Tracker*\", \"*leave*tracker*\", \"*Leave*Tracker*\", \"*LEAVE*TRACKER*\"]:\n        tracker_files.update(ws_path.glob(pattern))\n    xlsx_files = [f for f in tracker_files if f.suffix.lower() == \".xlsx\"]\n    \n    if not xlsx_files:\n        # Broaden: any xlsx with 'leave' or 'tracker' in name\n        all_xlsx = list(ws_path.glob(\"*.xlsx\"))\n        if not all_xlsx:\n            # Try recursively\n            all_xlsx = list(ws_path.rglob(\"*.xlsx\"))\n        if not all_xlsx:\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace.\"}\n        leave_files = [f for f in all_xlsx if \"leave\" in f.name.lower() or \"tracker\" in f.name.lower()]\n        if leave_files:\n            xlsx_files = leave_files\n        else:\n            xlsx_files = all_xlsx\n    \n    # Try each candidate file\n    for tracker_path in xlsx_files:\n        try:\n            wb = openpyxl.load_workbook(tracker_path, data_only=True)\n            # Try active sheet first, then all sheets\n            sheets_to_check = [wb.active] + [wb[s] for s in wb.sheetnames if wb[s] != wb.active]\n            for ws in sheets_to_check:\n                # Check N11 (row=11, column=14)\n                cell_value = ws.cell(row=11, column=14).value\n                if cell_value is not None:\n                    cell_str = str(cell_value).strip().lower()\n                    if \"p1\" in cell_str:\n                        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Cell N11 in '{tracker_path.name}' (sheet '{ws.title}') contains '{cell_value}' which includes 'P1'.\"}\n            \n            # Also search for LC-2026-0321 row and check its priority column\n            for ws in sheets_to_check:\n                target_row = None\n                priority_col = None\n                # Find header row and priority column\n                for row in range(1, min(ws.max_row + 1, 5)):\n                    for col in range(1, ws.max_column + 1):\n                        val = ws.cell(row=row, column=col).value\n                        if val and \"priority\" in str(val).strip().lower():\n                            priority_col = col\n                            break\n                    if priority_col:\n                        break\n                # Find row with LC-2026-0321\n                for row in range(1, ws.max_row + 1):\n                    for col in range(1, ws.max_column + 1):\n                        val = ws.cell(row=row, column=col).value\n                        if val and \"lc-2026-0321\" in str(val).strip().lower():\n                            target_row = row\n                            break\n                    if target_row:\n                        break\n                if target_row and priority_col:\n                    pval = ws.cell(row=target_row, column=priority_col).value\n                    if pval and \"p1\" in str(pval).strip().lower():\n                        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row {target_row} for LC-2026-0321 in '{tracker_path.name}' (sheet '{ws.title}') has priority '{pval}' which includes 'P1'.\"}\n            wb.close()\n        except Exception as e:\n            continue\n    \n    # Final failure - try to give useful info\n    try:\n        tracker_path = xlsx_files[0]\n        wb = openpyxl.load_workbook(tracker_path, data_only=True)\n        ws = wb.active\n        cell_value = ws.cell(row=11, column=14).value\n        wb.close()\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell N11 in '{tracker_path.name}' contains '{cell_value}', expected a value containing 'P1' (case-insensitive). Checked {len(xlsx_files)} file(s).\"}\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not verify. Error: {e}. Checked files: {[f.name for f in xlsx_files]}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_1776823960297",
    "sort_order": 6,
    "rubric_text": "In `slack_data.json`, there should be 3 messages in the `#benefits-team` channel.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    # Check external_services_path for slack_data.json\n    slack_path = None\n    if external_services_path:\n        candidate = Path(external_services_path) / \"slack_data.json\"\n        if candidate.exists():\n            slack_path = candidate\n    # Also check workspace_path\n    if not slack_path:\n        candidate = Path(workspace_path) / \"slack_data.json\"\n        if candidate.exists():\n            slack_path = candidate\n\n    if not slack_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found in workspace or external services path.\"}\n\n    try:\n        with open(slack_path, 'r') as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n\n    # Slack data format per the spec:\n    # {\n    #   \"channels\": { \"C001\": { \"id\": \"C001\", \"name\": \"general\", ... } },\n    #   \"messages\": { \"C001\": [ {message}, ... ] }\n    # }\n    # We need to find the channel ID for #benefits-team, then count messages.\n\n    benefits_channel_id = None\n    benefits_messages = None\n\n    # Strategy 1: Use the standard slack_data format (channels dict keyed by ID, messages dict keyed by channel ID)\n    if isinstance(data, dict):\n        channels = data.get('channels', {})\n        messages = data.get('messages', {})\n\n        # Find channel ID for benefits-team\n        if isinstance(channels, dict):\n            for ch_id, ch_info in channels.items():\n                if isinstance(ch_info, dict):\n                    ch_name = (ch_info.get('name', '') or '').strip().lower().replace('#', '')\n                    if ch_name == 'benefits-team':\n                        benefits_channel_id = ch_id\n                        break\n                elif isinstance(ch_info, str):\n                    # Maybe channels is {id: name}\n                    if ch_info.strip().lower().replace('#', '') == 'benefits-team':\n                        benefits_channel_id = ch_id\n                        break\n\n        # If we found the channel ID, get messages by channel ID\n        if benefits_channel_id and isinstance(messages, dict):\n            channel_msgs = messages.get(benefits_channel_id, [])\n            if isinstance(channel_msgs, list):\n                benefits_messages = channel_msgs\n\n        # Fallback: Try finding messages keyed directly by channel name\n        if benefits_messages is None and isinstance(messages, dict):\n            for key in messages:\n                normalized = key.strip().lower().replace('#', '')\n                if normalized == 'benefits-team':\n                    val = messages[key]\n                    if isinstance(val, list):\n                        benefits_messages = val\n                    break\n\n        # Fallback: top-level dict with channel names as keys\n        if benefits_messages is None:\n            for key in data:\n                normalized = key.strip().lower().replace('#', '')\n                if normalized == 'benefits-team':\n                    channel_data = data[key]\n                    if isinstance(channel_data, list):\n                        benefits_messages = channel_data\n                    elif isinstance(channel_data, dict) and 'messages' in channel_data:\n                        benefits_messages = channel_data['messages']\n                    break\n\n        # Fallback: channels is a list of dicts\n        if benefits_messages is None and isinstance(channels, list):\n            for ch in channels:\n                if isinstance(ch, dict):\n                    ch_name = (ch.get('name', '') or ch.get('channel_name', '') or '').strip().lower().replace('#', '')\n                    if ch_name == 'benefits-team':\n                        ch_id = ch.get('id', '')\n                        if ch_id and isinstance(messages, dict):\n                            benefits_messages = messages.get(ch_id, [])\n                        elif 'messages' in ch:\n                            benefits_messages = ch['messages']\n                        break\n\n        # Fallback: messages is a flat list with channel field\n        if benefits_messages is None and isinstance(messages, list):\n            benefits_messages = []\n            for msg in messages:\n                if isinstance(msg, dict):\n                    ch = (msg.get('channel', '') or msg.get('channel_name', '') or '').strip().lower().replace('#', '')\n                    if ch == 'benefits-team':\n                        benefits_messages.append(msg)\n\n    if benefits_messages is None:\n        debug_info = \"\"\n        if isinstance(data, dict):\n            debug_info = f\" Top-level keys: {list(data.keys())}.\"\n            channels = data.get('channels', {})\n            if isinstance(channels, dict):\n                channel_names = []\n                for ch_id, ch_info in channels.items():\n                    if isinstance(ch_info, dict):\n                        channel_names.append(f\"{ch_id}={ch_info.get('name', '?')}\")\n                    else:\n                        channel_names.append(f\"{ch_id}={ch_info}\")\n                debug_info += f\" Channels: {channel_names}.\"\n            msgs = data.get('messages', {})\n            if isinstance(msgs, dict):\n                debug_info += f\" Message keys: {list(msgs.keys())}.\"\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find #benefits-team channel in slack_data.json.{debug_info}\"}\n\n    count = len(benefits_messages)\n\n    if count == 3:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 3 messages in #benefits-team channel as expected.\"}\n    else:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 3 messages in #benefits-team channel, but found {count}. Messages: {json.dumps(benefits_messages, default=str)[:1000]}\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_1776826027518",
    "sort_order": 7,
    "rubric_text": "In `hfwa_balance_tracker.xlsx`, row 27 must be empty (not filled in with new employee data).",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    file_path = Path(workspace_path) / \"hfwa_balance_tracker.xlsx\"\n    if not file_path.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File 'hfwa_balance_tracker.xlsx' not found in workspace. Row 27 cannot have been filled in, so passing by default.\"}\n    \n    try:\n        wb = openpyxl.load_workbook(str(file_path), data_only=True)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'hfwa_balance_tracker.xlsx': {e}\"}\n    \n    # Check all sheets for row 27 content\n    for sheet_name in wb.sheetnames:\n        ws = wb[sheet_name]\n        row_27_values = []\n        for cell in ws[27]:\n            if cell.value is not None and str(cell.value).strip() != \"\":\n                row_27_values.append((cell.column_letter, cell.value))\n        \n        if row_27_values:\n            details = \"; \".join([f\"{col}: {val}\" for col, val in row_27_values])\n            return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 27 in sheet '{sheet_name}' of 'hfwa_balance_tracker.xlsx' is NOT empty. Found data: {details}\"}\n    \n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 27 in 'hfwa_balance_tracker.xlsx' is empty across all sheets, as required.\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_1776826059275",
    "sort_order": 8,
    "rubric_text": "In `hours_worked_log.xlsx`, row 27 must be empty (not filled in with new employee data).",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    file_path = Path(workspace_path) / \"hours_worked_log.xlsx\"\n    if not file_path.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File 'hours_worked_log.xlsx' does not exist, so row 27 is trivially empty (no new employee data was written there).\"}\n    \n    try:\n        wb = openpyxl.load_workbook(file_path, data_only=True)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'hours_worked_log.xlsx': {e}\"}\n    \n    # Check all sheets for row 27 content\n    for sheet_name in wb.sheetnames:\n        ws = wb[sheet_name]\n        row_27_values = []\n        for cell in ws[27]:\n            if cell.value is not None and str(cell.value).strip() != \"\":\n                row_27_values.append((cell.column_letter, cell.value))\n        \n        if row_27_values:\n            details = \", \".join([f\"{col}={val}\" for col, val in row_27_values])\n            return {\n                \"pass\": False,\n                \"score\": 0.0,\n                \"feedback\": f\"Row 27 in sheet '{sheet_name}' of 'hours_worked_log.xlsx' is NOT empty. Found values: {details}\"\n            }\n    \n    wb.close()\n    return {\n        \"pass\": True,\n        \"score\": 1.0,\n        \"feedback\": \"Row 27 in 'hours_worked_log.xlsx' is empty across all sheets — no new employee data was written there.\"\n    }\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_1776826254479",
    "sort_order": 9,
    "rubric_text": "In `mailbox.json`, there should be no emails after email_id 40, and email_id 40 should be addressed to 'tomas.guerrero@ridgelinegear.com'.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    mailbox_path = None\n\n    # Try external_services_path first, then workspace_path\n    if external_services_path:\n        candidate = Path(external_services_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n\n    if not mailbox_path:\n        candidate = Path(workspace_path) / \"mailbox.json\"\n        if candidate.exists():\n            mailbox_path = candidate\n\n    if not mailbox_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No mailbox.json found in either external_services_path or workspace_path.\"}\n\n    try:\n        with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n\n    emails = data.get(\"emails\", [])\n    total_emails = len(emails)\n    issues = []\n    score = 1.0\n\n    # Build a lookup by email_id (could be int or string)\n    emails_by_id = {}\n    max_email_id = None\n    for email in emails:\n        eid = email.get(\"email_id\", email.get(\"id\", None))\n        if eid is not None:\n            try:\n                eid_int = int(eid)\n            except (ValueError, TypeError):\n                eid_int = None\n            if eid_int is not None:\n                emails_by_id[eid_int] = email\n                if max_email_id is None or eid_int > max_email_id:\n                    max_email_id = eid_int\n\n    # Check: no emails after email_id 40\n    emails_after_40 = [eid for eid in emails_by_id if eid > 40]\n    if emails_after_40:\n        issues.append(f\"Found emails with email_id > 40: {sorted(emails_after_40)}. There should be no emails after email_id 40.\")\n        score = 0.0\n\n    # Check: email_id 40 exists and is addressed to tomas.guerrero@ridgelinegear.com\n    expected_recipient = \"tomas.guerrero@ridgelinegear.com\"\n    if 40 in emails_by_id:\n        email_40 = emails_by_id[40]\n        to_addr = email_40.get(\"to_addr\", email_40.get(\"to\", \"\"))\n        # Normalize: accept string or list\n        if isinstance(to_addr, str):\n            to_addrs = [to_addr.strip().lower()]\n        elif isinstance(to_addr, list):\n            to_addrs = [a.strip().lower() for a in to_addr]\n        else:\n            to_addrs = []\n\n        if expected_recipient not in to_addrs:\n            issues.append(f\"Email with email_id 40 is addressed to '{to_addr}', expected '{expected_recipient}'.\")\n            score = 0.0\n    else:\n        issues.append(f\"No email with email_id 40 found. Available email_ids: {sorted(emails_by_id.keys())[-10:] if emails_by_id else 'none'}\")\n        score = 0.0\n\n    if issues:\n        return {\"pass\": False, \"score\": score, \"feedback\": \"Issues found: \" + \"; \".join(issues)}\n    else:\n        email_40 = emails_by_id[40]\n        to_addr = email_40.get(\"to_addr\", email_40.get(\"to\", \"\"))\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json has no emails after email_id 40 (max email_id is {max_email_id}). Email_id 40 is addressed to '{to_addr}', which matches the expected recipient '{expected_recipient}'.\"}\n",
    "criterion_type": "expected_output"
  }
]