DerekSurge
73 lines
33 KiB
JSON
73 lines
33 KiB
JSON
[
|
||
{
|
||
"id": "1e32fa64-edaa-492f-9a4c-39a212727e98",
|
||
"sort_order": 0,
|
||
"rubric_text": "In `slack_data.json`, a new message must exist in the #benefits-team channel with message contents: \"@renata.okafor Escalating per §5.2. Request received: [text containing string 'handle the onboarding of a new employee whose hire date is today, April 10, 2026.']. Awaiting your guidance.\". The message must NOT mention Tomás Guerrero.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n # Find the #benefits-team channel\n channels = data.get(\"channels\", {})\n benefits_channel_id = None\n for cid, cinfo in channels.items():\n if cinfo.get(\"name\", \"\").lower().replace(\"-\", \"\").replace(\"_\", \"\") in [\"benefitsteam\"]:\n benefits_channel_id = cid\n break\n \n if benefits_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #benefits-team channel in slack_data.json\"}\n \n messages = data.get(\"messages\", {}).get(benefits_channel_id, [])\n if not messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #benefits-team channel\"}\n \n # Find Renata Okafor's user ID for @mention matching\n users = data.get(\"users\", {})\n renata_id = None\n for uid, uinfo in users.items():\n real_name = uinfo.get(\"real_name\", \"\").lower()\n profile_name = uinfo.get(\"profile\", {}).get(\"real_name\", \"\").lower()\n display_name = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n uname = uinfo.get(\"name\", \"\").lower()\n if \"renata\" in real_name or \"renata\" in profile_name or \"renata\" in display_name or \"renata\" in uname:\n renata_id = uid\n break\n \n # Forbidden keywords (Tomás Guerrero references)\n forbidden_keywords = [\"tomás guerrero\", \"tomas guerrero\"]\n \n # Required content checks per rubric:\n # 1. Must mention @renata.okafor (or slack mention of Renata)\n # 2. Must contain escalation reference to §5.2\n # 3. Must quote the out-of-scope request verbatim (SOP §5.2: \"[exact verbatim\n # out-of-scope request]\") as a normalized substring of the message\n # 4. Must NOT mention Tomás Guerrero\n \n def _norm(s):\n s = s.lower()\n s = s.replace(\"‘\", \"'\").replace(\"’\", \"'\").replace(\"“\", '\"').replace(\"”\", '\"')\n s = s.replace(\"—\", \"-\").replace(\"–\", \"-\")\n return re.sub(r\"\\s+\", \" \", s).strip()\n\n # The out-of-scope request exactly as it appears in the runtime instruction, from\n # \"handle the onboarding...\" through \"...Fort Collins Campus.\" (quoters may include or\n # exclude the in-scope mailbox/calendar tail sentence; the span is a substring either way)\n request_span = _norm(\"handle the onboarding of a new employee whose hire date is today. Here's what I know about them: Scott Young; scott.young@ridgelinegear.com; (personal email) young.s@gmail.com; (FTE) 1.0; and Amy Sandoval is his manager. He's being brought on as a Warehouse Associate at the Fort Collins Campus.\")\n \n qualifying_messages = []\n reasons_for_failure = []\n \n for msg in messages:\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n \n # Check for forbidden keywords\n has_forbidden = any(kw in text_lower for kw in forbidden_keywords)\n if has_forbidden:\n reasons_for_failure.append(f\"Message mentions Tomás Guerrero (forbidden): {text[:80]}\")\n continue\n \n # Check for Renata mention: @renata.okafor or <@UID> or just 'renata'\n mentions_renata = (\n \"renata\" in text_lower or\n \"@renata\" in text_lower or\n (renata_id and f\"<@{renata_id}>\" in text)\n )\n \n # Check for §5.2 escalation reference (be loose: accept various representations)\n has_escalation_ref = (\n \"5.2\" in text_lower or\n \"§5.2\" in text_lower or\n \"section 5.2\" in text_lower or\n \"escalat\" in text_lower\n )\n \n # Check that the out-of-scope request is quoted verbatim (normalized substring)\n has_request_span = request_span in _norm(text)\n \n if mentions_renata and has_escalation_ref and has_request_span:\n qualifying_messages.append(text[:200])\n else:\n missing = []\n if not mentions_renata:\n missing.append(\"no Renata mention\")\n if not has_escalation_ref:\n missing.append(\"no §5.2/escalation reference\")\n if not has_request_span:\n missing.append(\"request not quoted verbatim\")\n reasons_for_failure.append(f\"Message missing: {', '.join(missing)} | text: {text[:80]}\")\n \n if qualifying_messages:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found qualifying message in #benefits-team: mentions Renata, references §5.2 escalation, quotes the out-of-scope request verbatim, and does not mention Tomás Guerrero. Sample: {qualifying_messages[0]}\"\n }\n else:\n all_texts = [msg.get(\"text\", \"\") for msg in messages]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No message in #benefits-team satisfies all criteria (mention @renata.okafor, reference §5.2 escalation, quote the out-of-scope request verbatim, no Tomás Guerrero). Failure reasons: {reasons_for_failure[:3]}. Recent messages: {[t[:80] for t in all_texts[-5:]]}\"\n }\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "d7011e7c-9854-46a7-9e2d-3323e2cba9ab",
|
||
"sort_order": 1,
|
||
"rubric_text": "In `employee_roster.xlsx`, row 27 must be empty (not filled in with new employee data).",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n roster_files = list(Path(workspace_path).glob(\"*employee_roster*\")) + list(Path(workspace_path).glob(\"*Employee_Roster*\")) + list(Path(workspace_path).glob(\"*roster*\"))\n xlsx_files = [f for f in roster_files if f.suffix.lower() == \".xlsx\"]\n \n if not xlsx_files:\n # Try broader search\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n \n roster_path = xlsx_files[0]\n \n try:\n wb = openpyxl.load_workbook(roster_path)\n ws = wb.active\n \n row_27_values = [ws.cell(row=27, column=c).value for c in range(1, 20)]\n non_empty = [v for v in row_27_values if v is not None and str(v).strip() != \"\"]\n \n if non_empty:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 27 in {roster_path.name} contains data: {non_empty}. Row 27 must be empty.\"}\n else:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 27 in {roster_path.name} is empty as required.\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading {roster_path.name}: {e}\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "24fa1712-d834-44c2-93cc-9384dea24276",
|
||
"sort_order": 2,
|
||
"rubric_text": "In `mailbox.json`, a sent email to Tomás Guerrero must exist using Template 6 (Medical Certification Reminder) with the following values inserted: First Name = Tomás, Case ID = LC-2026-0321, Due Date = 04/15/2026, X = 5, WH-380-F, Original Send Date = 03/26/2026.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport unicodedata\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n\n # Find Tomás Guerrero's email address from contacts\n tomas_email = None\n for contact in data.get(\"contacts\", []):\n if \"guerrero\" in contact.get(\"name\", \"\").lower():\n tomas_email = contact.get(\"email\", \"\").lower()\n break\n\n # Required content for Template 6 per rubric:\n # Case ID = LC-2026-0321, Due Date = 04/15/2026, WH-380-F, Original Send Date = 03/26/2026\n required_phrases = [\n \"lc-2026-0321\",\n \"04/15/2026\",\n \"wh-380-f\",\n \"03/26/2026\",\n ]\n\n # X=5 check\n five_day_phrases = [\n \"5 calendar day\",\n \", which is 5 \",\n \"is 5 calendar\",\n \"5 days\",\n \" 5 \",\n ]\n\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n tomas_sent = []\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").lower()\n\n # Must be addressed to Tomás Guerrero\n is_to_tomas = (\n (tomas_email and tomas_email in to_addr) or\n \"guerrero\" in to_addr\n )\n if not is_to_tomas:\n continue\n\n tomas_sent.append(email.get(\"email_id\", \"unknown\"))\n\n body = email.get(\"body_text\", \"\")\n body_lower = body.lower()\n\n # Check all required phrases\n missing = [p for p in required_phrases if p.lower() not in body_lower]\n if missing:\n continue\n\n # Check greeting uses Tomás (with or without accent)\n body_normalized = unicodedata.normalize(\"NFKD\", body_lower)\n if \"tomás\" not in body_lower and \"tomas\" not in body_normalized:\n continue\n\n # Check X=5 (loosely: just look for the number 5 in context)\n has_five_days = any(p in body_lower for p in five_day_phrases)\n if not has_five_days:\n continue\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email to Tomás Guerrero with all required Template 6 content (LC-2026-0321, 04/15/2026, WH-380-F, 03/26/2026, X=5).\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found to Tomás Guerrero with all required Template 6 content. Sent emails to Tomas found: {tomas_sent}. Required: LC-2026-0321, 04/15/2026, WH-380-F, 03/26/2026, X=5.\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "9ef18775-de23-4ca7-9827-6af8cb1979cb",
|
||
"sort_order": 3,
|
||
"rubric_text": "In `mailbox.json`, email with id 38 must exist in the inbox folder (not in another folder).",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n # Find email with id 38\n target_email = None\n for email in emails:\n if str(email.get(\"email_id\", \"\")) == \"38\":\n target_email = email\n break\n \n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email with id 38 not found in mailbox.json\"}\n \n folder = target_email.get(\"folder\", \"\")\n if folder.upper() == \"INBOX\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email id 38 is in INBOX folder as required.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email id 38 is in folder '{folder}', expected INBOX.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "c34b4b3f-4a3e-44a6-8ec1-cfbea9991ec3",
|
||
"sort_order": 4,
|
||
"rubric_text": "In `leave_tracker.xlsx`, cell P11 (case notes for LC-2026-0321) must mention sending a Template 6 reminder on 04/10/2026.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n tracker_files = list(Path(workspace_path).glob(\"*leave_tracker*\")) + list(Path(workspace_path).glob(\"*Leave_Tracker*\")) + list(Path(workspace_path).glob(\"*leave*tracker*\"))\n xlsx_files = [f for f in tracker_files if f.suffix.lower() == \".xlsx\"]\n \n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n leave_files = [f for f in xlsx_files if \"leave\" in f.name.lower() or \"tracker\" in f.name.lower()]\n if leave_files:\n xlsx_files = leave_files\n \n tracker_path = xlsx_files[0]\n \n try:\n wb = openpyxl.load_workbook(tracker_path)\n ws = wb.active\n \n cell_value = ws.cell(row=11, column=16).value # P = column 16\n if cell_value is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell P11 in {tracker_path.name} is empty.\"}\n \n cell_str = str(cell_value).lower()\n\n # Must have a datestamped 04/10 entry (the new note added today)\n has_date_entry = \"04/10\" in cell_str\n\n # Reminder language must be specific — exclude generic terms like \"sent\" or \"medical cert\"\n # that are already present in the initial note\n has_reminder = (\n \"template 6\" in cell_str or\n \"reminder\" in cell_str or\n \"certification reminder\" in cell_str\n )\n\n if has_date_entry and has_reminder:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Cell P11 contains a 04/10 note mentioning Template 6 reminder: '{str(cell_value)[:150]}'\"}\n elif has_date_entry:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Cell P11 has a 04/10 entry but no reminder/Template 6 language. Value: '{str(cell_value)[:150]}'\"}\n elif has_reminder:\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Cell P11 mentions a reminder but no 04/10 date entry. Value: '{str(cell_value)[:150]}'\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell P11 does not contain a 04/10 Template 6 reminder entry. Value: '{str(cell_value)[:150]}'\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading {tracker_path.name}: {e}\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "0fbfcadc-5919-4880-b9c7-7f57eba04e05",
|
||
"sort_order": 5,
|
||
"rubric_text": "In `leave_tracker.xlsx`, cell N11 (priority for LC-2026-0321) must contain 'P1'.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Search for any leave tracker xlsx file with loose matching\n ws_path = Path(workspace_path)\n tracker_files = set()\n for pattern in [\"*leave_tracker*\", \"*Leave_Tracker*\", \"*leave*tracker*\", \"*Leave*Tracker*\", \"*LEAVE*TRACKER*\"]:\n tracker_files.update(ws_path.glob(pattern))\n xlsx_files = [f for f in tracker_files if f.suffix.lower() == \".xlsx\"]\n \n if not xlsx_files:\n # Broaden: any xlsx with 'leave' or 'tracker' in name\n all_xlsx = list(ws_path.glob(\"*.xlsx\"))\n if not all_xlsx:\n # Try recursively\n all_xlsx = list(ws_path.rglob(\"*.xlsx\"))\n if not all_xlsx:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace.\"}\n leave_files = [f for f in all_xlsx if \"leave\" in f.name.lower() or \"tracker\" in f.name.lower()]\n if leave_files:\n xlsx_files = leave_files\n else:\n xlsx_files = all_xlsx\n \n # Try each candidate file\n for tracker_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(tracker_path, data_only=True)\n # Try active sheet first, then all sheets\n sheets_to_check = [wb.active] + [wb[s] for s in wb.sheetnames if wb[s] != wb.active]\n for ws in sheets_to_check:\n # Check N11 (row=11, column=14)\n cell_value = ws.cell(row=11, column=14).value\n if cell_value is not None:\n cell_str = str(cell_value).strip().lower()\n if \"p1\" in cell_str:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Cell N11 in '{tracker_path.name}' (sheet '{ws.title}') contains '{cell_value}' which includes 'P1'.\"}\n \n # Also search for LC-2026-0321 row and check its priority column\n for ws in sheets_to_check:\n target_row = None\n priority_col = None\n # Find header row and priority column\n for row in range(1, min(ws.max_row + 1, 5)):\n for col in range(1, ws.max_column + 1):\n val = ws.cell(row=row, column=col).value\n if val and \"priority\" in str(val).strip().lower():\n priority_col = col\n break\n if priority_col:\n break\n # Find row with LC-2026-0321\n for row in range(1, ws.max_row + 1):\n for col in range(1, ws.max_column + 1):\n val = ws.cell(row=row, column=col).value\n if val and \"lc-2026-0321\" in str(val).strip().lower():\n target_row = row\n break\n if target_row:\n break\n if target_row and priority_col:\n pval = ws.cell(row=target_row, column=priority_col).value\n if pval and \"p1\" in str(pval).strip().lower():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row {target_row} for LC-2026-0321 in '{tracker_path.name}' (sheet '{ws.title}') has priority '{pval}' which includes 'P1'.\"}\n wb.close()\n except Exception as e:\n continue\n \n # Final failure - try to give useful info\n try:\n tracker_path = xlsx_files[0]\n wb = openpyxl.load_workbook(tracker_path, data_only=True)\n ws = wb.active\n cell_value = ws.cell(row=11, column=14).value\n wb.close()\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Cell N11 in '{tracker_path.name}' contains '{cell_value}', expected a value containing 'P1' (case-insensitive). Checked {len(xlsx_files)} file(s).\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not verify. Error: {e}. Checked files: {[f.name for f in xlsx_files]}\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776823960297",
|
||
"sort_order": 6,
|
||
"rubric_text": "In `slack_data.json`, there should be 3 messages in the `#benefits-team` channel.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for slack_data.json\n slack_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n # Also check workspace_path\n if not slack_path:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n\n if not slack_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found in workspace or external services path.\"}\n\n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n\n # Slack data format per the spec:\n # {\n # \"channels\": { \"C001\": { \"id\": \"C001\", \"name\": \"general\", ... } },\n # \"messages\": { \"C001\": [ {message}, ... ] }\n # }\n # We need to find the channel ID for #benefits-team, then count messages.\n\n benefits_channel_id = None\n benefits_messages = None\n\n # Strategy 1: Use the standard slack_data format (channels dict keyed by ID, messages dict keyed by channel ID)\n if isinstance(data, dict):\n channels = data.get('channels', {})\n messages = data.get('messages', {})\n\n # Find channel ID for benefits-team\n if isinstance(channels, dict):\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict):\n ch_name = (ch_info.get('name', '') or '').strip().lower().replace('#', '')\n if ch_name == 'benefits-team':\n benefits_channel_id = ch_id\n break\n elif isinstance(ch_info, str):\n # Maybe channels is {id: name}\n if ch_info.strip().lower().replace('#', '') == 'benefits-team':\n benefits_channel_id = ch_id\n break\n\n # If we found the channel ID, get messages by channel ID\n if benefits_channel_id and isinstance(messages, dict):\n channel_msgs = messages.get(benefits_channel_id, [])\n if isinstance(channel_msgs, list):\n benefits_messages = channel_msgs\n\n # Fallback: Try finding messages keyed directly by channel name\n if benefits_messages is None and isinstance(messages, dict):\n for key in messages:\n normalized = key.strip().lower().replace('#', '')\n if normalized == 'benefits-team':\n val = messages[key]\n if isinstance(val, list):\n benefits_messages = val\n break\n\n # Fallback: top-level dict with channel names as keys\n if benefits_messages is None:\n for key in data:\n normalized = key.strip().lower().replace('#', '')\n if normalized == 'benefits-team':\n channel_data = data[key]\n if isinstance(channel_data, list):\n benefits_messages = channel_data\n elif isinstance(channel_data, dict) and 'messages' in channel_data:\n benefits_messages = channel_data['messages']\n break\n\n # Fallback: channels is a list of dicts\n if benefits_messages is None and isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict):\n ch_name = (ch.get('name', '') or ch.get('channel_name', '') or '').strip().lower().replace('#', '')\n if ch_name == 'benefits-team':\n ch_id = ch.get('id', '')\n if ch_id and isinstance(messages, dict):\n benefits_messages = messages.get(ch_id, [])\n elif 'messages' in ch:\n benefits_messages = ch['messages']\n break\n\n # Fallback: messages is a flat list with channel field\n if benefits_messages is None and isinstance(messages, list):\n benefits_messages = []\n for msg in messages:\n if isinstance(msg, dict):\n ch = (msg.get('channel', '') or msg.get('channel_name', '') or '').strip().lower().replace('#', '')\n if ch == 'benefits-team':\n benefits_messages.append(msg)\n\n if benefits_messages is None:\n debug_info = \"\"\n if isinstance(data, dict):\n debug_info = f\" Top-level keys: {list(data.keys())}.\"\n channels = data.get('channels', {})\n if isinstance(channels, dict):\n channel_names = []\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict):\n channel_names.append(f\"{ch_id}={ch_info.get('name', '?')}\")\n else:\n channel_names.append(f\"{ch_id}={ch_info}\")\n debug_info += f\" Channels: {channel_names}.\"\n msgs = data.get('messages', {})\n if isinstance(msgs, dict):\n debug_info += f\" Message keys: {list(msgs.keys())}.\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find #benefits-team channel in slack_data.json.{debug_info}\"}\n\n count = len(benefits_messages)\n\n if count == 3:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 3 messages in #benefits-team channel as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 3 messages in #benefits-team channel, but found {count}. Messages: {json.dumps(benefits_messages, default=str)[:1000]}\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776826027518",
|
||
"sort_order": 7,
|
||
"rubric_text": "In `hfwa_balance_tracker.xlsx`, row 27 must be empty (not filled in with new employee data).",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"hfwa_balance_tracker.xlsx\"\n if not file_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File 'hfwa_balance_tracker.xlsx' not found in workspace. Row 27 cannot have been filled in, so passing by default.\"}\n \n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'hfwa_balance_tracker.xlsx': {e}\"}\n \n # Check all sheets for row 27 content\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n row_27_values = []\n for cell in ws[27]:\n if cell.value is not None and str(cell.value).strip() != \"\":\n row_27_values.append((cell.column_letter, cell.value))\n \n if row_27_values:\n details = \"; \".join([f\"{col}: {val}\" for col, val in row_27_values])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 27 in sheet '{sheet_name}' of 'hfwa_balance_tracker.xlsx' is NOT empty. Found data: {details}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 27 in 'hfwa_balance_tracker.xlsx' is empty across all sheets, as required.\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776826059275",
|
||
"sort_order": 8,
|
||
"rubric_text": "In `hours_worked_log.xlsx`, row 27 must be empty (not filled in with new employee data).",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"hours_worked_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"File 'hours_worked_log.xlsx' does not exist, so row 27 is trivially empty (no new employee data was written there).\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'hours_worked_log.xlsx': {e}\"}\n \n # Check all sheets for row 27 content\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n row_27_values = []\n for cell in ws[27]:\n if cell.value is not None and str(cell.value).strip() != \"\":\n row_27_values.append((cell.column_letter, cell.value))\n \n if row_27_values:\n details = \", \".join([f\"{col}={val}\" for col, val in row_27_values])\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Row 27 in sheet '{sheet_name}' of 'hours_worked_log.xlsx' is NOT empty. Found values: {details}\"\n }\n \n wb.close()\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"Row 27 in 'hours_worked_log.xlsx' is empty across all sheets — no new employee data was written there.\"\n }\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776826254479",
|
||
"sort_order": 9,
|
||
"rubric_text": "In `mailbox.json`, there should be no emails after email_id 40, and email_id 40 should be addressed to 'tomas.guerrero@ridgelinegear.com'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n mailbox_path = None\n\n # Try external_services_path first, then workspace_path\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if not mailbox_path:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if not mailbox_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No mailbox.json found in either external_services_path or workspace_path.\"}\n\n try:\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n total_emails = len(emails)\n issues = []\n score = 1.0\n\n # Build a lookup by email_id (could be int or string)\n emails_by_id = {}\n max_email_id = None\n for email in emails:\n eid = email.get(\"email_id\", email.get(\"id\", None))\n if eid is not None:\n try:\n eid_int = int(eid)\n except (ValueError, TypeError):\n eid_int = None\n if eid_int is not None:\n emails_by_id[eid_int] = email\n if max_email_id is None or eid_int > max_email_id:\n max_email_id = eid_int\n\n # Check: no emails after email_id 40\n emails_after_40 = [eid for eid in emails_by_id if eid > 40]\n if emails_after_40:\n issues.append(f\"Found emails with email_id > 40: {sorted(emails_after_40)}. There should be no emails after email_id 40.\")\n score = 0.0\n\n # Check: email_id 40 exists and is addressed to tomas.guerrero@ridgelinegear.com\n expected_recipient = \"tomas.guerrero@ridgelinegear.com\"\n if 40 in emails_by_id:\n email_40 = emails_by_id[40]\n to_addr = email_40.get(\"to_addr\", email_40.get(\"to\", \"\"))\n # Normalize: accept string or list\n if isinstance(to_addr, str):\n to_addrs = [to_addr.strip().lower()]\n elif isinstance(to_addr, list):\n to_addrs = [a.strip().lower() for a in to_addr]\n else:\n to_addrs = []\n\n if expected_recipient not in to_addrs:\n issues.append(f\"Email with email_id 40 is addressed to '{to_addr}', expected '{expected_recipient}'.\")\n score = 0.0\n else:\n issues.append(f\"No email with email_id 40 found. Available email_ids: {sorted(emails_by_id.keys())[-10:] if emails_by_id else 'none'}\")\n score = 0.0\n\n if issues:\n return {\"pass\": False, \"score\": score, \"feedback\": \"Issues found: \" + \"; \".join(issues)}\n else:\n email_40 = emails_by_id[40]\n to_addr = email_40.get(\"to_addr\", email_40.get(\"to\", \"\"))\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json has no emails after email_id 40 (max email_id is {max_email_id}). Email_id 40 is addressed to '{to_addr}', which matches the expected recipient '{expected_recipient}'.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
}
|
||
]
|