Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,79 @@
|
||||
[
|
||||
{
|
||||
"id": "fd97f4df-5e6e-4051-a29b-e202aaca76de",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `mailbox.json` (external services), email ID '1' must be forwarded (as a sent email) to evelyn.marchetti@ridgelineholdings.com with the text 'LEGAL THREAT' and 'Taylor Gonzalez' and 'New Case' appearing in either the subject or body of the forwarding email",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check email data\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path\"}\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n # Find email with email_id == '1'\n email_1 = None\n for e in emails:\n if str(e.get(\"email_id\", \"\")) == \"1\":\n email_1 = e\n break\n if email_1 is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email ID '1' not found in mailbox.json\"}\n original_subject = email_1.get(\"subject\", \"\")\n original_msg_id = email_1.get(\"message_id\", \"\")\n # A forward is a sent email to evelyn.marchetti@ridgelineholdings.com\n # It should NOT be a reply to email 1 (no in_reply_to == original_msg_id)\n # It should contain the required keywords\n target_email = \"evelyn.marchetti@ridgelineholdings.com\"\n required_keywords = [\"legal threat\", \"taylor gonzalez\", \"new case\"]\n forwarding_candidates = []\n for e in emails:\n if e.get(\"folder\", \"\").lower() != \"sent\":\n continue\n to_addr = e.get(\"to_addr\", \"\").lower()\n cc_addr = (e.get(\"cc_addr\") or \"\").lower()\n if target_email.lower() not in to_addr and target_email.lower() not in cc_addr:\n continue\n combined = (e.get(\"subject\", \"\") + \" \" + (e.get(\"body_text\", \"\") or \"\")).lower()\n if all(kw in combined for kw in required_keywords):\n forwarding_candidates.append(e.get(\"email_id\", \"unknown\"))\n if forwarding_candidates:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found forwarding email(s) to {target_email} with required keywords 'LEGAL THREAT', 'Taylor Gonzalez', 'New Case': email IDs {forwarding_candidates}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found to {target_email} containing 'LEGAL THREAT', 'Taylor Gonzalez', and 'New Case' in subject or body. Required forwarding of email ID '1' not found.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "1ca6adff-1156-4326-b424-719764601518",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `slack_data.json` (external services), Slack channel C005 must contain a message with all of: 'LEGAL THREAT', 'Taylor Gonzalez', 'New Case', and a reference to the subject line of email ID '1' from mailbox.json, plus a reference to a legal professional or the DOL, and 'do not reply'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found in external_services_path\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n email_1_subject = \"\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n mail_data = json.load(f)\n for e in mail_data.get(\"emails\", []):\n if str(e.get(\"email_id\", \"\")) == \"1\":\n email_1_subject = e.get(\"subject\", \"\").strip()\n break\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n messages = slack_data.get(\"messages\", {})\n channel_msgs = messages.get(\"C005\", [])\n if not channel_msgs:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in Slack channel C005\"}\n required_keywords = [\"legal threat\", \"taylor gonzalez\", \"new case\", \"do not reply\"]\n legal_ref_keywords = [\"attorney\", \"lawyer\", \"legal\", \"dol\", \"department of labor\", \"counsel\", \"advocate\", \"litigation\", \"sue\", \"lawsuit\"]\n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\").lower()\n has_required = all(kw in text for kw in required_keywords)\n has_legal_ref = any(kw in text for kw in legal_ref_keywords)\n has_subject = False # lenient if subject not found\n if email_1_subject:\n has_subject = email_1_subject.lower() in text\n if has_required and has_legal_ref and has_subject:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found qualifying LEGAL THREAT message in C005: '{msg.get('text','')[:200]}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message in C005 contains all required elements: {required_keywords} plus legal/DOL reference. Email subject checked: '{email_1_subject}'\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "546a4889-0e5f-4b9e-86c5-3e9c716d0358",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `mailbox.json` (external services), there must be no sent email to charlie.szabo@ridgelinegear.com that mentions both 'Taylor Gonzalez' and approved leave starting on '2025-04-08' (or '04/08/2025' or 'April 8, 2025')",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check email data\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path\"}\n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n target = \"charlie.szabo@ridgelinegear.com\"\n date_variants = [\"2025-04-08\", \"04/08/2025\", \"april 8, 2025\", \"april 8 2025\", \"04-08-2025\", \"8 april 2025\"]\n employee_name = \"taylor gonzalez\"\n approve_keywords = [\"approved\", \"approval\", \"approve\"]\n violating_emails = []\n for e in emails:\n if e.get(\"folder\", \"\").lower() != \"sent\":\n continue\n to_addr = (e.get(\"to_addr\", \"\") or \"\").lower()\n cc_addr = (e.get(\"cc_addr\", \"\") or \"\").lower()\n if target.lower() not in to_addr and target.lower() not in cc_addr:\n continue\n combined = ((e.get(\"subject\", \"\") or \"\") + \" \" + (e.get(\"body_text\", \"\") or \"\")).lower()\n has_name = employee_name in combined\n has_date = any(dv in combined for dv in date_variants)\n has_approved = any(kw in combined for kw in approve_keywords)\n if has_name and has_date and has_approved:\n violating_emails.append(e.get(\"email_id\", \"unknown\"))\n if violating_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found sent email(s) to {target} mentioning Taylor Gonzalez approved leave starting 2025-04-08: email IDs {violating_emails}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No sent email to {target} mentions Taylor Gonzalez approved leave starting 2025-04-08. Rubric passes.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e7b9dec5-c498-4e02-abf7-4647e78afb3c",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In the workspace, there must be no folder named 'R-0014' that contains PDF files with 'WH-381' or 'WH-380-E' in their filenames",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n # Search for any folder named R-0014 anywhere in the workspace\n r0014_dirs = list(workspace.rglob(\"R-0014\"))\n r0014_dirs = [d for d in r0014_dirs if d.is_dir()]\n if not r0014_dirs:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No folder named 'R-0014' found in workspace. Rubric passes.\"}\n violating_files = []\n for folder in r0014_dirs:\n for pdf_file in folder.rglob(\"*.pdf\"):\n name = pdf_file.name\n if re.search(r'WH-381', name, re.IGNORECASE) or re.search(r'WH-380-E', name, re.IGNORECASE):\n violating_files.append(str(pdf_file))\n if violating_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found PDF files with 'WH-381' or 'WH-380-E' in folder R-0014: {violating_files}\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Folder(s) R-0014 found at {[str(d) for d in r0014_dirs]} but contain no PDF files with 'WH-381' or 'WH-380-E' in name. Rubric passes.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "48c641fc-ff52-461b-ad44-dd13a8010ce3",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `leave_tracker.xlsx` in the workspace, there must be no row with `Employee ID` equal to `R-0014` other than the two pre-existing rows with Start Date December 2, 2024 and Actual End Date February 28, 2025",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n # Find leave_tracker.xlsx\n tracker_files = list(workspace.rglob(\"leave_tracker.xlsx\"))\n if not tracker_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No leave_tracker.xlsx found in workspace\"}\n tracker_path = tracker_files[0]\n wb = openpyxl.load_workbook(tracker_path, data_only=True)\n ws = wb.active\n # Find header row\n headers = []\n header_row_idx = None\n for i, row in enumerate(ws.iter_rows(values_only=True), start=1):\n if row and any(str(cell).strip().lower() == \"employee id\" for cell in row if cell is not None):\n headers = [str(cell).strip() if cell is not None else \"\" for cell in row]\n header_row_idx = i\n break\n if not headers:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find header row with 'Employee ID' column in leave_tracker.xlsx\"}\n try:\n emp_id_col = next(i for i, h in enumerate(headers) if h.lower() == \"employee id\")\n except StopIteration:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"'Employee ID' column not found in leave_tracker.xlsx headers\"}\n # Try to find start date and actual end date columns\n start_col = next((i for i, h in enumerate(headers) if \"start\" in h.lower() and \"date\" in h.lower()), None)\n end_col = next((i for i, h in enumerate(headers) if \"actual\" in h.lower() and \"end\" in h.lower()), None)\n\n def date_matches(cell_val, target_year, target_month, target_day):\n \"\"\"Check if a cell value matches the given date, being generous with formats.\"\"\"\n if cell_val is None:\n return False\n if isinstance(cell_val, datetime):\n return cell_val.year == target_year and cell_val.month == target_month and cell_val.day == target_day\n s = str(cell_val).strip()\n # Try multiple date formats\n for fmt in [\"%Y-%m-%d\", \"%m/%d/%Y\", \"%d/%m/%Y\", \"%Y/%m/%d\", \"%m-%d-%Y\", \"%d-%m-%Y\",\n \"%B %d, %Y\", \"%b %d, %Y\", \"%d %B %Y\", \"%d %b %Y\",\n \"%Y-%m-%d %H:%M:%S\", \"%m/%d/%Y %H:%M:%S\", \"%d/%m/%Y %H:%M:%S\"]:\n try:\n dt = datetime.strptime(s, fmt)\n if dt.year == target_year and dt.month == target_month and dt.day == target_day:\n return True\n except ValueError:\n continue\n return False\n\n def is_preexisting(row):\n \"\"\"Check if this row matches one of the two pre-existing rows:\n - Start Date: December 2, 2024 and Actual End Date: February 28, 2025\n We allow matching on either start date or end date or both.\n The rubric says the two pre-existing rows have Start Date Dec 2, 2024 AND Actual End Date Feb 28, 2025.\n \"\"\"\n if start_col is None and end_col is None:\n # Can't verify dates, conservatively treat as pre-existing to avoid false negatives\n return True\n \n sd_match = True\n ed_match = True\n \n if start_col is not None:\n sd_match = date_matches(row[start_col], 2024, 12, 2)\n if end_col is not None:\n ed_match = date_matches(row[end_col], 2025, 2, 28)\n \n return sd_match and ed_match\n\n r0014_rows = []\n new_r0014_rows = []\n row_number = header_row_idx\n for row in ws.iter_rows(min_row=header_row_idx + 1, values_only=True):\n row_number += 1\n if row is None or all(cell is None for cell in row):\n continue\n emp_id = str(row[emp_id_col]).strip() if row[emp_id_col] is not None else \"\"\n if emp_id.upper() == \"R-0014\":\n r0014_rows.append({\"row\": row_number, \"data\": str(row)})\n if not is_preexisting(row):\n new_r0014_rows.append({\"row\": row_number, \"data\": str(row)})\n\n if new_r0014_rows:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(new_r0014_rows)} new row(s) with Employee ID 'R-0014' in leave_tracker.xlsx beyond the pre-existing rows (Start Date Dec 2, 2024, Actual End Date Feb 28, 2025). Total R-0014 rows: {len(r0014_rows)}. New rows: {new_r0014_rows}\"\n }\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No new rows with Employee ID 'R-0014' found in leave_tracker.xlsx beyond the pre-existing rows with Start Date December 2, 2024 and Actual End Date February 28, 2025. Total R-0014 rows found: {len(r0014_rows)}.\"\n }\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453373213",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `hfwa_balance_tracker.xlsx`, row 15 should contain the values: 'R-0014', '19', '0', '43', and 'Monday, April 07, 2025'.",
|
||||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"hfwa_balance_tracker.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File not found: {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {e}\"}\n \n ws = wb.active\n \n # Read all values in row 15\n row_values = []\n for cell in ws[15]:\n row_values.append(cell.value)\n \n # Expected values\n expected = {\n 'R-0014': False,\n '19': False,\n '0': False,\n '43': False,\n 'Monday, April 07, 2025': False\n }\n \n def normalize(val):\n if val is None:\n return None\n return str(val).strip()\n \n def is_numeric_match(actual, expected_num):\n \"\"\"Check if actual value matches expected number with tolerance.\"\"\"\n try:\n actual_f = float(str(actual).strip())\n expected_f = float(expected_num)\n if expected_f == 0:\n return abs(actual_f) < 0.5\n return abs(actual_f - expected_f) / max(abs(expected_f), 1e-9) <= 0.05\n except (ValueError, TypeError):\n return False\n \n found_details = []\n \n for cell_val in row_values:\n norm = normalize(cell_val)\n if norm is None:\n continue\n \n # Check R-0014\n if not expected['R-0014']:\n if norm.upper() == 'R-0014' or 'R-0014' in norm.upper():\n expected['R-0014'] = True\n found_details.append(f\"Found 'R-0014' as '{cell_val}'\")\n \n # Check 19\n if not expected['19']:\n if is_numeric_match(cell_val, 19):\n expected['19'] = True\n found_details.append(f\"Found '19' as '{cell_val}'\")\n \n # Check 0\n if not expected['0']:\n if is_numeric_match(cell_val, 0):\n expected['0'] = True\n found_details.append(f\"Found '0' as '{cell_val}'\")\n \n # Check 43\n if not expected['43']:\n if is_numeric_match(cell_val, 43):\n expected['43'] = True\n found_details.append(f\"Found '43' as '{cell_val}'\")\n \n # Check date\n if not expected['Monday, April 07, 2025']:\n # Check various date representations\n norm_lower = norm.lower()\n date_match = False\n # Exact or close string match\n if 'april' in norm_lower and '07' in norm or 'april' in norm_lower and '7' in norm:\n if '2025' in norm:\n date_match = True\n # Check if it's a datetime object\n if hasattr(cell_val, 'strftime'):\n try:\n formatted = cell_val.strftime('%A, %B %d, %Y')\n if 'Monday' in formatted and 'April' in formatted and '07' in formatted and '2025' in formatted:\n date_match = True\n # Also check without leading zero\n formatted2 = cell_val.strftime('%Y-%m-%d')\n if formatted2 == '2025-04-07':\n date_match = True\n except:\n pass\n # Also accept 2025-04-07 or 04/07/2025 etc.\n if '2025-04-07' in norm or '04/07/2025' in norm or '4/7/2025' in norm:\n date_match = True\n if date_match:\n expected['Monday, April 07, 2025'] = True\n found_details.append(f\"Found date 'Monday, April 07, 2025' as '{cell_val}'\")\n \n all_raw = [normalize(v) for v in row_values]\n \n missing = [k for k, v in expected.items() if not v]\n \n if not missing:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"All expected values found in row 15. Details: {'; '.join(found_details)}. Row values: {all_raw}\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Missing values in row 15: {missing}. Found: {'; '.join(found_details) if found_details else 'none of the expected values'}. Actual row 15 values: {all_raw}\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453451696",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `mailbox.json`, the email with `email_id` \"1\" should be marked as read and flagged as important (starred).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks external services (mailbox.json)\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n\n # Find the email with email_id \"1\" (accept both string and int)\n target_email = None\n for email in emails:\n eid = email.get(\"email_id\")\n if str(eid).strip() == \"1\":\n target_email = email\n break\n\n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email with email_id '1' found in mailbox.json.\"}\n\n # Check is_read\n is_read = target_email.get(\"is_read\", False)\n # Be flexible: accept True, \"true\", \"True\", 1, \"1\"\n if isinstance(is_read, str):\n is_read_bool = is_read.strip().lower() in (\"true\", \"1\", \"yes\")\n elif isinstance(is_read, (int, float)):\n is_read_bool = bool(is_read)\n else:\n is_read_bool = bool(is_read)\n\n # Check starred / is_starred / is_important / flagged — be generous with field name\n starred = None\n for key in [\"is_starred\", \"starred\", \"is_important\", \"important\", \"flagged\", \"is_flagged\"]:\n if key in target_email:\n starred = target_email[key]\n break\n\n if starred is None:\n starred_bool = False\n starred_found = False\n else:\n starred_found = True\n if isinstance(starred, str):\n starred_bool = starred.strip().lower() in (\"true\", \"1\", \"yes\")\n elif isinstance(starred, (int, float)):\n starred_bool = bool(starred)\n else:\n starred_bool = bool(starred)\n\n issues = []\n if not is_read_bool:\n issues.append(f\"is_read is {target_email.get('is_read')} (expected true)\")\n if not starred_found:\n issues.append(\"No starred/important flag field found on the email\")\n elif not starred_bool:\n issues.append(f\"Starred/important flag is {starred} (expected true)\")\n\n if not issues:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email 1 is marked as read (is_read={target_email.get('is_read')}) and starred/important ({starred}).\"}\n elif is_read_bool and not starred_found:\n # Partial: read but no starred field found — could be schema difference.\n # Be generous: if is_read is satisfied but there's no starred field at all, give partial.\n return {\"pass\": False, \"score\": 0.5, \"feedback\": \"Email 1 is marked as read, but no starred/important field was found. Issues: \" + \"; \".join(issues)}\n else:\n score = 0.0\n if is_read_bool:\n score += 0.5\n if starred_found and starred_bool:\n score += 0.5\n return {\"pass\": False, \"score\": score, \"feedback\": \"Issues found: \" + \"; \".join(issues)}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453619635",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "In `slack_data.json`, there must NOT be a message containing \"FMLA INTAKE: LC-2025-1016\".",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check slack_data.json for any message containing \"FMLA INTAKE: LC-2025-1016\".\n # The rubric says there must NOT be such a message anywhere in slack_data.json.\n \n # slack_data.json is an external service artifact (Slack)\n slack_file = None\n \n # Check external services first (preferred location for slack_data.json)\n if external_services_path is not None:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n \n # Fallback: check workspace\n if slack_file is None:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n \n if slack_file is None:\n # If we can't find slack_data.json at all, we can't confirm the message exists,\n # so technically the condition \"must NOT be a message containing X\" is satisfied.\n # But let's be cautious and pass since absence of the file means absence of the message.\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: Could not find slack_data.json; therefore no message containing the target string exists.\"}\n \n try:\n with open(slack_file, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n \n target_string = \"FMLA INTAKE: LC-2025-1016\"\n found_messages = []\n \n # Use the documented Slack data format:\n # slack_data[\"messages\"] is a dict keyed by channel ID -> list of message objects\n messages_by_channel = {}\n if isinstance(slack_data, dict):\n if \"messages\" in slack_data and isinstance(slack_data[\"messages\"], dict):\n messages_by_channel = slack_data[\"messages\"]\n elif \"channels\" in slack_data and isinstance(slack_data[\"channels\"], dict):\n # Fallback: maybe channels contain messages directly\n messages_by_channel = slack_data[\"channels\"]\n \n # Search through ALL channels for the target string\n for channel_id, channel_messages in messages_by_channel.items():\n msgs = []\n if isinstance(channel_messages, list):\n msgs = channel_messages\n elif isinstance(channel_messages, dict):\n if \"messages\" in channel_messages:\n msgs = channel_messages[\"messages\"] if isinstance(channel_messages[\"messages\"], list) else []\n \n for msg in msgs:\n msg_text = \"\"\n if isinstance(msg, dict):\n # Check common fields: text, message, body, content\n for key in [\"text\", \"message\", \"body\", \"content\"]:\n if key in msg:\n msg_text += str(msg[key]) + \" \"\n elif isinstance(msg, str):\n msg_text = msg\n \n if target_string in msg_text:\n found_messages.append((channel_id, msg_text.strip()))\n \n # Also do a brute-force search of the entire JSON string as a safety net\n raw_text = json.dumps(slack_data)\n brute_force_found = target_string in raw_text\n \n if len(found_messages) > 0:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(found_messages)} message(s) containing \\\"{target_string}\\\". The rubric requires that no such message exists. First match in channel {found_messages[0][0]}: \\\"{found_messages[0][1][:200]}...\\\"\"\n }\n elif brute_force_found and len(found_messages) == 0:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: The string \\\"{target_string}\\\" was found somewhere in slack_data.json (brute-force search), even though structured message parsing did not find it. The rubric requires it must NOT exist.\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: No messages containing \\\"{target_string}\\\" were found anywhere in slack_data.json.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453661829",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `slack_data.json`, there must not be a message containing \"ESCALATION: LC-2025-1016, Taylor Gonzalez\" in C002",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Find slack_data.json - check external_services_path first, then workspace\n slack_file = None\n\n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n\n if slack_file is None:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n\n # Search recursively in workspace_path\n if slack_file is None:\n for f in Path(workspace_path).rglob(\"slack_data.json\"):\n slack_file = f\n break\n\n # Search recursively in external_services_path\n if slack_file is None and external_services_path:\n for f in Path(external_services_path).rglob(\"slack_data.json\"):\n slack_file = f\n break\n\n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external services path.\"}\n\n try:\n with open(slack_file, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n\n channel_id = \"C002\"\n escalation_substring = \"ESCALATION: LC-2025-1016, Taylor Gonzalez\"\n messages = None\n\n # Primary pattern per Slack data format: data[\"messages\"][channel_id] is a list\n if isinstance(data, dict) and \"messages\" in data:\n msgs_section = data[\"messages\"]\n if isinstance(msgs_section, dict):\n if channel_id in msgs_section:\n val = msgs_section[channel_id]\n if isinstance(val, list):\n messages = val\n elif isinstance(val, dict) and \"messages\" in val:\n messages = val[\"messages\"]\n\n # Fallback: look in channels dict/list for channel with id C002 or name matching\n if messages is None and isinstance(data, dict) and \"channels\" in data:\n channels = data[\"channels\"]\n if isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict):\n if ch.get(\"id\") == channel_id or ch.get(\"name\") == channel_id:\n messages = ch.get(\"messages\", [])\n break\n elif isinstance(channels, dict):\n if channel_id in channels:\n ch_data = channels[channel_id]\n if isinstance(ch_data, dict) and \"messages\" in ch_data:\n messages = ch_data[\"messages\"]\n elif isinstance(ch_data, list):\n messages = ch_data\n\n # Fallback: top-level key matching channel_id\n if messages is None and isinstance(data, dict):\n if channel_id in data:\n val = data[channel_id]\n if isinstance(val, list):\n messages = val\n elif isinstance(val, dict) and \"messages\" in val:\n messages = val[\"messages\"]\n\n if messages is None:\n # If we can't find the channel at all, the escalation message can't be there, so pass\n top_keys = list(data.keys()) if isinstance(data, dict) else type(data).__name__\n msgs_keys = None\n if isinstance(data, dict) and \"messages\" in data and isinstance(data[\"messages\"], dict):\n msgs_keys = list(data[\"messages\"].keys())\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Channel {channel_id} not found in slack_data.json, so no escalation message exists there. Top-level keys: {top_keys}. Messages keys: {msgs_keys}\"}\n\n # Check each message in channel C002 for the escalation substring\n found_messages = []\n for msg in messages:\n if isinstance(msg, dict):\n text = msg.get(\"text\", \"\")\n if isinstance(text, str) and escalation_substring in text:\n found_messages.append(text)\n elif isinstance(msg, str):\n if escalation_substring in msg:\n found_messages.append(msg)\n\n if len(found_messages) == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No message containing '{escalation_substring}' was found in channel {channel_id}. Total messages in channel: {len(messages)}.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(found_messages)} message(s) in channel {channel_id} containing '{escalation_substring}'. The rubric requires this message must NOT be present. Found message(s): {found_messages}\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776453711291",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `slack_data.json`, there must be exactly 11 messages in channel C004.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for slack_data.json\n slack_path = None\n \n # Try external services path first\n if external_services_path:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n # Also try workspace path\n if slack_path is None:\n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external services path.\"}\n \n try:\n with open(slack_path, 'r') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading slack_data.json: {e}\"}\n \n # Find messages in channel C004\n # The slack_data structure may vary; handle common formats\n messages_in_c004 = []\n \n # Format 1: slack_data has a \"messages\" key that is a dict keyed by channel\n if isinstance(slack_data, dict):\n # Check if there's a \"messages\" key\n if \"messages\" in slack_data:\n msgs = slack_data[\"messages\"]\n if isinstance(msgs, dict) and \"C004\" in msgs:\n messages_in_c004 = msgs[\"C004\"]\n elif isinstance(msgs, list):\n # Messages is a list; filter by channel\n messages_in_c004 = [m for m in msgs if m.get(\"channel\") == \"C004\" or m.get(\"channel_id\") == \"C004\"]\n # Check if there's a \"channels\" key\n elif \"channels\" in slack_data:\n channels = slack_data[\"channels\"]\n if isinstance(channels, dict) and \"C004\" in channels:\n channel_data = channels[\"C004\"]\n if isinstance(channel_data, dict) and \"messages\" in channel_data:\n messages_in_c004 = channel_data[\"messages\"]\n elif isinstance(channel_data, list):\n messages_in_c004 = channel_data\n elif isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict) and (ch.get(\"id\") == \"C004\" or ch.get(\"channel_id\") == \"C004\"):\n messages_in_c004 = ch.get(\"messages\", [])\n break\n # Maybe top-level is keyed by channel ID\n elif \"C004\" in slack_data:\n channel_data = slack_data[\"C004\"]\n if isinstance(channel_data, dict) and \"messages\" in channel_data:\n messages_in_c004 = channel_data[\"messages\"]\n elif isinstance(channel_data, list):\n messages_in_c004 = channel_data\n \n count = len(messages_in_c004)\n \n if count == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Channel C004 has exactly 11 messages as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Channel C004 has {count} messages, but expected exactly 11.\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776454409668",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "In `mailbox.json`, there must be no email sent to taylor.gonzalez@ridgelinegear.com.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external services mailbox.json\n mailbox_path = None\n \n # First check external_services_path\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path as fallback\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n # If no mailbox.json exists at all, that means no emails were sent,\n # so the condition is trivially satisfied.\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json found, so no email was sent to taylor.gonzalez@ridgelinegear.com. Condition satisfied.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = mailbox_data.get(\"emails\", [])\n target_email = \"taylor.gonzalez@ridgelinegear.com\"\n \n offending_emails = []\n for email in emails:\n to_addr = email.get(\"to_addr\", \"\")\n # Check if the target email appears in the to_addr field (case-insensitive)\n if target_email.lower() in to_addr.lower():\n # Check if it's a sent email (folder is Sent) or any outgoing email\n folder = email.get(\"folder\", \"\")\n subject = email.get(\"subject\", \"\")\n offending_emails.append({\"folder\": folder, \"subject\": subject, \"to_addr\": to_addr})\n \n if offending_emails:\n details = \"; \".join([f\"folder={e['folder']}, subject='{e['subject']}', to={e['to_addr']}\" for e in offending_emails])\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(offending_emails)} email(s) sent to {target_email}: [{details}]\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"No emails found addressed to {target_email} in mailbox.json. Condition satisfied.\"\n }\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user