52 lines
26 KiB
JSON
52 lines
26 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "1cb3faea-7dcd-4223-ba22-82ef885c61ca",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In `mailbox.json`, there must be a sent email from a.costa@mojavecrest.com to asolis@griffinmedical.org with subject line exactly 'Mojave Crest Request KPX658320914 - Information Needed'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n target_subject = \"Mojave Crest Request KPX658320914 - Information Needed\"\n found_sent = []\n for email in emails:\n folder = (email.get(\"folder\") or \"\").lower().strip()\n if folder != \"sent\":\n continue\n from_addr = (email.get(\"from_addr\") or email.get(\"from\") or \"\").lower().strip()\n to_addr_raw = (email.get(\"to_addr\") or email.get(\"to\") or \"\").strip()\n subject = (email.get(\"subject\") or \"\").strip()\n # Normalize from address\n if from_addr != \"a.costa@mojavecrest.com\":\n continue\n # Normalize to address - handle various unicode issues\n to_lower = to_addr_raw.lower().strip()\n to_normalized = to_lower.replace(\"\\u02c6\", \"i\").replace(\"\\ufb01\", \"fi\").replace(\"\\ufb00\", \"ff\")\n to_match = (\"asolis@griffinmedical.org\" in to_normalized or\n \"asolis@grif\" in to_normalized)\n if not to_match:\n continue\n # Check subject - be lenient with whitespace and dashes\n subject_match = (subject == target_subject)\n if not subject_match:\n # Try normalizing dashes and whitespace\n import re\n norm_subject = re.sub(r'[\\s\\u2013\\u2014]+', ' ', subject).strip()\n norm_target = re.sub(r'[\\s\\u2013\\u2014]+', ' ', target_subject).strip()\n # Also try replacing em/en dashes with hyphen\n norm_subject2 = re.sub(r'[\\u2013\\u2014]', '-', subject).strip()\n subject_match = (norm_subject == norm_target or norm_subject2 == target_subject)\n if not subject_match:\n found_sent.append({\"subject\": subject, \"to\": to_addr_raw})\n continue\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found sent email from a.costa@mojavecrest.com to {to_addr_raw} with subject '{subject}'.\"}\n # Build detailed failure feedback\n if found_sent:\n details = \"; \".join([f\"subject='{e['subject']}', to='{e['to']}'\" for e in found_sent[:5]])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found sent emails from a.costa@mojavecrest.com to matching recipient but subject didn't match. Expected subject: '{target_subject}'. Found: {details}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No sent email found from a.costa@mojavecrest.com to asolis@griffinmedical.org with subject '{target_subject}'. Total emails in mailbox: {len(emails)}.\"}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "9b96be64-d380-48ad-8fa9-56149892311c",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In `mailbox.json`, the sent email to asolis@griffinmedical.org must have a body that: (1) addresses 'Avery Solis' or 'Dr. Avery Solis', (2) contains all required boilerplate text verbatim (key phrases: 'In order to complete our review', 'within 15 calendar days', 'portal.mojavecrest.com', 'Ari Costa', 'Appeals & Grievances Coordinator', 'Mojave Crest Assurance Company', 'Phone: (702) 555-0148') OR attaches the letter as a file.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n\n # Find the target sent email to asolis@griffinmedical.org\n target_email = None\n for email in emails:\n if email.get(\"folder\", \"\").lower() != \"sent\":\n continue\n to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").lower().strip()\n # Normalize unicode characters that may appear\n to_normalized = to_addr.replace(\"\\u02c6\", \"i\").replace(\"\\ufb01\", \"fi\")\n if \"asolis@griffinmedical.org\" in to_normalized or \"asolis@grif\" in to_normalized:\n target_email = email\n break\n\n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find a sent email to asolis@griffinmedical.org in mailbox.json. Sent emails found: \" + str([(e.get('to_addr') or e.get('to','')) for e in emails if e.get('folder','').lower() == 'sent'])}\n\n body = (target_email.get(\"body_text\") or \"\").strip()\n if not body:\n body = (target_email.get(\"body\") or \"\").strip()\n\n # Check for attachments - the rubric says body must have boilerplate OR attach the letter as a file\n attachments = target_email.get(\"attachments\", []) or []\n has_attachment = len(attachments) > 0\n\n # Also check if there's an attachment field as a string\n if not has_attachment:\n att_str = target_email.get(\"attachment\", \"\")\n if att_str:\n has_attachment = True\n\n failures = []\n found_details = []\n\n # (1) Check salutation: addresses 'Avery Solis' or 'Dr. Avery Solis'\n # Check both body and attachment content. Be loose: check case-insensitively too.\n salutation_ok = False\n if re.search(r'Avery\\s+Solis', body, re.IGNORECASE):\n salutation_ok = True\n \n # If there's an attachment, try to read it from workspace to check content\n attachment_content = \"\"\n if has_attachment:\n for att in (attachments if isinstance(attachments, list) else [attachments]):\n att_name = att if isinstance(att, str) else (att.get(\"filename\") or att.get(\"name\") or \"\")\n if att_name:\n # Try to find the file in workspace\n att_path = Path(workspace_path) / att_name\n if att_path.exists():\n try:\n if att_name.lower().endswith(\".pdf\"):\n import pdfplumber\n with pdfplumber.open(att_path) as pdf:\n for page in pdf.pages:\n txt = page.extract_text()\n if txt:\n attachment_content += txt + \"\\n\"\n elif att_name.lower().endswith(\".docx\"):\n import docx\n doc = docx.Document(att_path)\n for para in doc.paragraphs:\n attachment_content += para.text + \"\\n\"\n else:\n attachment_content += att_path.read_text(errors='ignore')\n except Exception:\n pass\n # Also check subdirectories\n else:\n for found in Path(workspace_path).rglob(att_name):\n try:\n if att_name.lower().endswith(\".pdf\"):\n impor
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "0c65195f-0c61-4c8c-8bd6-c9f09f21c823",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "A PDF file named exactly '20260512_KPX658320914_Correspondence.pdf' must exist in the workspace.",
|
||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n pdf_path = Path(workspace_path) / \"20260512_KPX658320914_Correspondence.pdf\"\n if pdf_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found '20260512_KPX658320914_Correspondence.pdf' in workspace.\"}\n # Also check subdirectories one level deep\n matches = list(Path(workspace_path).rglob(\"20260512_KPX658320914_Correspondence.pdf\"))\n if matches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found '20260512_KPX658320914_Correspondence.pdf' at {matches[0]}.\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File '20260512_KPX658320914_Correspondence.pdf' not found in workspace.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ef1720bf-390a-44c1-a9e5-5b4e637f7516",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In file `20260512_KPX658320914_Correspondence.pdf`, the document must include: date 5/12/2026 (or equivalent), addressee 'Jules Arden', address '2675 Westwind Terrace, Las Vegas, NV 89146', salutation addressing 'Jules Arden', all required boilerplate phrases ('In order to complete our review', 'within 15 calendar days', 'portal.mojavecrest.com', 'Ari Costa', 'Appeals & Grievances Coordinator', 'Mojave Crest Assurance Company', '(702) 555-0148').",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import pdfplumber\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"pdfplumber not available\"}\n\n matches = list(Path(workspace_path).rglob(\"20260512_KPX658320914_Correspondence.pdf\"))\n if not matches:\n # Also try looser glob patterns\n matches = list(Path(workspace_path).rglob(\"*KPX658320914*Correspondence*.pdf\"))\n if not matches:\n matches = list(Path(workspace_path).rglob(\"*Correspondence*.pdf\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"File '20260512_KPX658320914_Correspondence.pdf' (or similar) not found anywhere in workspace.\"}\n\n pdf_path = matches[0]\n try:\n with pdfplumber.open(str(pdf_path)) as pdf:\n text = \"\"\n for page in pdf.pages:\n text += (page.extract_text() or \"\") + \"\\n\"\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading PDF '{pdf_path.name}': {e}\"}\n\n if not text.strip():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"PDF '{pdf_path.name}' appears to have no extractable text.\"}\n\n failures = []\n found_items = []\n\n # Normalize whitespace for easier matching\n norm_text = re.sub(r'\\s+', ' ', text)\n\n # Check date (accept various formats: 5/12/2026, 05/12/2026, May 12, 2026, 2026-05-12)\n date_patterns = [r\"5/12/2026\", r\"05/12/2026\", r\"May\\s*12,?\\s*2026\", r\"2026-05-12\", r\"12\\s+May\\s+2026\"]\n date_found = any(re.search(p, text, re.IGNORECASE) for p in date_patterns)\n if not date_found:\n # Also check normalized text\n date_found = any(re.search(p, norm_text, re.IGNORECASE) for p in date_patterns)\n if date_found:\n found_items.append(\"date 5/12/2026\")\n else:\n failures.append(\"Date 5/12/2026 (or equivalent) not found in document\")\n\n # Check addressee Jules Arden\n if \"Jules Arden\" in text or \"Jules Arden\" in norm_text or re.search(r'Jules\\s+Arden', text, re.IGNORECASE):\n found_items.append(\"addressee 'Jules Arden'\")\n else:\n failures.append(\"Addressee 'Jules Arden' not found\")\n\n # Check address components\n addr_issues = []\n if re.search(r'2675\\s+Westwind\\s+Terrace', text, re.IGNORECASE) or \"2675 Westwind Terrace\" in norm_text:\n found_items.append(\"'2675 Westwind Terrace'\")\n else:\n addr_issues.append(\"'2675 Westwind Terrace' not found\")\n\n if re.search(r'Las\\s+Vegas', text, re.IGNORECASE):\n found_items.append(\"'Las Vegas'\")\n else:\n addr_issues.append(\"City 'Las Vegas' not found\")\n\n if re.search(r'NV', text):\n found_items.append(\"'NV'\")\n # NV not strictly required by rubric as separate check, but part of address\n\n if \"89146\" in text:\n found_items.append(\"ZIP '89146'\")\n else:\n addr_issues.append(\"ZIP '89146' not found\")\n\n if addr_issues:\n failures.append(\"Address issues: \" + \"; \".join(addr_issues))\n\n # Check salutation addressing Jules Arden\n salutation_patterns = [\n r'Dear\\s+Jules\\s+Arden',\n r'Dear\\s+Mr\\.?\\s+Arden',\n r'Dear\\s+Ms\\.?\\s+Arden',\n r'Dear\\s+Mrs\\.?\\s+Arden',\n r'Dear\\s+Dr\\.?\\s+Arden',\n r'Dear\\s+Mx\\.?\\s+Arden',\n r'Dear\\s+Jules',\n ]\n salutation_ok = any(re.search(p, text, re.IGNORECASE) for p in salutation_patterns)\n if not salutation_ok:\n salutation_ok = any(re.search(p, norm_text, re.IGNORECASE) for p in salutation_patterns)\n if salutation_ok:\n found_items.append(\"salutation addressing Jules Arden\")\n else:\n failures.append(\"Salutation addressing 'Jules Arden' not found (expected 'Dear Jules Arden' or similar)\")\n\n # Check required boilerplate phrases per rubric\n boilerplate_checks = [\n (\"In order to
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "559f22cc-3db0-40a2-a652-6c39e79cafb9",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In `appeals_log_master.xlsx`, there must be no row with Member ID 'KPX658320914'.",
|
||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n # Search for the file\n matches = list(Path(workspace_path).rglob(\"appeals_log_master.xlsx\"))\n if not matches:\n # If the file doesn't exist at all, there's no entry - pass\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"appeals_log_master.xlsx not found; no KPX658320914 entry exists.\"}\n xlsx_path = matches[0]\n try:\n wb = openpyxl.load_workbook(str(xlsx_path), read_only=True, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading appeals_log_master.xlsx: {e}\"}\n member_id = \"KPX658320914\"\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row in ws.iter_rows(values_only=True):\n for cell in row:\n if cell is not None and str(cell).strip() == member_id:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found Member ID '{member_id}' in appeals_log_master.xlsx sheet '{sheet_name}'.\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No entry for Member ID '{member_id}' found in appeals_log_master.xlsx.\"}",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "13f466fd-60c9-4b1b-8f58-d34b5bbd1d92",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In `mailbox.json`, no sent email to asolis@griffinmedical.org or Jules Arden must reference an appeal acknowledgment or contain a Case ID number (i.e., no email body should contain phrases like 'Case ID', 'case number', 'appeal has been received', 'appeal is being processed').",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n forbidden_patterns = [\n r\"case\\s*id\",\n r\"case\\s*number\",\n r\"appeal has been received\",\n r\"appeal is being processed\",\n r\"your appeal\",\n r\"case\\s*#\"\n ]\n violations = []\n for email in emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n body = (email.get(\"body_text\") or \"\").lower()\n # Check emails directed to Avery Solis or Jules Arden\n recipient_match = (\n \"asolis\" in to_addr or \n \"griffinmedical\" in to_addr or\n \"jules arden\" in (email.get(\"to_addr\") or \"\").lower() or\n \"jules arden\" in body\n )\n if not recipient_match:\n continue\n for pattern in forbidden_patterns:\n if re.search(pattern, body, re.IGNORECASE) or re.search(pattern, subject, re.IGNORECASE):\n violations.append(f\"Email to '{email.get('to_addr')}' contains forbidden pattern '{pattern}'\")\n break\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(violations)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No emails to Avery Solis or Jules Arden contain appeal acknowledgment or Case ID references.\"}",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "e563f6f7-37c9-4b1d-bed6-2ea3176ce159",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In `slack_data.json`, there must be no Slack message in any channel that is directed to/mentions @jhenning or j.henning@mojavecrest.com regarding member KPX658320914. Also, in `mailbox.json`, there must be no email to j.henning@mojavecrest.com mentioning 'KPX658320914'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack or mailbox data\"}\n member_id = \"KPX658320914\"\n violations = []\n # Check Slack\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n messages = slack_data.get(\"messages\", {})\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n text = (msg.get(\"text\") or \"\").lower()\n # Check if message mentions jhenning or j.henning@mojavecrest.com AND references the member ID\n mentions_henning = (\"jhenning\" in text or \"j.henning@mojavecrest.com\" in text or \"@jhenning\" in text)\n mentions_member = member_id.lower() in text\n if mentions_henning and mentions_member:\n violations.append(f\"Slack message in channel {channel_id} mentions Dr. Henning and {member_id}: '{msg.get('text', '')[:100]}'\")\n else:\n pass # slack_data.json not present is acceptable\n # Check email to Dr. Jules Henning\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n mail_data = json.load(f)\n emails = mail_data.get(\"emails\", [])\n for email in emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n if \"j.henning@mojavecrest.com\" in to_addr or \"jhenning@mojavecrest\" in to_addr:\n if member_id.lower() in body or member_id.lower() in subject:\n violations.append(f\"Email to j.henning@mojavecrest.com references {member_id}\")\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(violations)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No Slack messages or emails to Dr. Henning reference {member_id}.\"}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
}
|
||
|
|
]
|