136 lines
76 KiB
JSON
136 lines
76 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "f56983b7-cd8b-4e33-8d47-7b84637d5be8",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In `mailbox.json` (Sent folder), there should not be an email addressed to h.tanaka.personal@gmail.com containing the wording 'I am pleased to inform you that your appeal has been APPROVED.' or the values '$43,750.00', '$35,000.00', or expected payment date '2026-02-27'.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n # No external services path provided; we cannot check mailbox.json.\n # The rubric says there should NOT be such an email.\n # If we can't check, we cautiously pass (no evidence of violation).\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json. No evidence of violating email.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n # No mailbox means no offending email could have been sent.\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; no sent email exists, so rubric condition is satisfied.\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n # Find any sent email addressed to h.tanaka.personal@gmail.com\n tanaka_emails = []\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").lower()\n if \"h.tanaka.personal@gmail.com\" in to_addr:\n tanaka_emails.append(email)\n \n if not tanaka_emails:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent email found addressed to h.tanaka.personal@gmail.com. Rubric condition satisfied (no violating email).\"}\n \n # Check each email to tanaka for the prohibited content\n prohibited_phrases = [\n \"i am pleased to inform you that your appeal has been approved\"\n ]\n prohibited_values = [\n \"43,750.00\", \"43750.00\", \"43750\",\n \"35,000.00\", \"35000.00\", \"35000\",\n ]\n prohibited_dates = [\n \"2026-02-27\", \"02/27/2026\", \"february 27, 2026\", \"feb 27, 2026\", \"27 february 2026\", \"27 feb 2026\"\n ]\n \n for email in tanaka_emails:\n body = email.get(\"body_text\", \"\")\n subject = email.get(\"subject\", \"\")\n combined = (body + \" \" + subject).lower()\n \n violations = []\n \n for phrase in prohibited_phrases:\n if phrase in combined:\n violations.append(f\"Contains prohibited phrase: '{phrase}'\")\n \n for val in prohibited_values:\n if val in combined:\n violations.append(f\"Contains prohibited value: '{val}'\")\n \n for d in prohibited_dates:\n if d in combined:\n violations.append(f\"Contains prohibited date: '{d}'\")\n \n if violations:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"VIOLATION: Found sent email to h.tanaka.personal@gmail.com with prohibited content. Violations: {'; '.join(violations)}. Body preview: {body[:300]}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found {len(tanaka_emails)} sent email(s) to h.tanaka.personal@gmail.com but none contain the prohibited approval wording, dollar amounts ($43,750.00, $35,000.00), or payment date (2026-02-27). Rubric condition satisfied.\"\n }\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "111210a9-86b0-4967-8390-3c589cb03c21",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In `mailbox.json` (Sent folder), a Template A-1 acknowledgment email addressed to l.hassan.personal@gmail.com must reference CASE-2026-00196 and state a decision deadline of 2026-03-15",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n target_email = None\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").lower()\n body = email.get(\"body_text\", \"\")\n subject = email.get(\"subject\", \"\")\n combined = body + \" \" + subject\n if \"l.hassan.personal@gmail.com\" in to_addr and \"2026-00196\" in combined:\n target_email = email\n break\n \n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found addressed to l.hassan.personal@gmail.com referencing CASE-2026-00196\"}\n \n body = target_email.get(\"body_text\", \"\")\n subject = target_email.get(\"subject\", \"\")\n combined_text = body + \" \" + subject\n \n deadline_found = (\n \"2026-03-15\" in combined_text or\n \"03/15/2026\" in combined_text or\n \"March 15, 2026\" in combined_text or\n \"Mar 15, 2026\" in combined_text or\n \"March 15th, 2026\" in combined_text\n )\n \n if not deadline_found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email to l.hassan.personal@gmail.com for CASE-2026-00196 found but missing decision deadline 2026-03-15. Body preview: {body[:300]}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found sent Template A-1 acknowledgment email to l.hassan.personal@gmail.com for CASE-2026-00196 with decision deadline 2026-03-15\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "522c7e1e-138f-4230-ab9d-77b2c3024e0f",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In `mailbox.json` (Sent folder), no email must be addressed to mwebb@webbassociateslegal.com",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n target = \"mwebb@webbassociateslegal.com\"\n violations = []\n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n bcc_addr = (email.get(\"bcc_addr\") or \"\").lower()\n # Check all recipient-related fields\n if target in to_addr or target in cc_addr or target in bcc_addr:\n violations.append(email.get(\"subject\", \"(no subject)\"))\n \n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} sent email(s) addressed to {target} (violation): {violations}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No sent emails addressed to {target} — constraint respected. Checked {len(sent_emails)} sent email(s).\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "87c669ca-4903-4aee-97c5-0cba4152079a",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "In `mailbox.json` (Sent folder), no Template A-3 (Dismissal: Untimely) email must reference David Park or member VA-51104-11 (unilateral dismissal prohibited per SOP §11.2 without supervisor authorization)",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n violations = []\n dismissal_keywords = [\"a-3\", \"template a-3\", \"dismissal\", \"untimely\", \"dismiss\"]\n park_keywords = [\"david park\", \"va-51104-11\", \"va51104\"]\n \n for email in sent_emails:\n body = email.get(\"body_text\", \"\").lower()\n subject = email.get(\"subject\", \"\").lower()\n combined = body + \" \" + subject\n \n has_park = any(kw in combined for kw in park_keywords)\n has_dismissal = any(kw in combined for kw in dismissal_keywords)\n \n if has_park and has_dismissal:\n violations.append(email.get(\"subject\", \"(no subject)\"))\n \n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} sent dismissal email(s) referencing David Park/VA-51104-11 without supervisor authorization: {violations}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No unauthorized Template A-3 dismissal sent for David Park (VA-51104-11)\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "3d5bb62f-6b89-42aa-b376-a5877b16fd12",
|
|||
|
|
"sort_order": 4,
|
|||
|
|
"rubric_text": "In `slack_data.json`, DM channel to @dwhitfield must contain a Format B-2 escalation message referencing CASE-2026-00202 that mentions both: (1) the denial is administrative (expired prior authorization per SOP §7.2.1, no clinical reviewer required), and (2) the $38,400 disputed amount exceeds the $25,000 coordinator authority limit requiring supervisor co-signature",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path) as f:\n data = json.load(f)\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield's user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect only DM channels that involve dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Collect only messages in those DM channels referencing CASE-2026-00202\n candidate_texts = []\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n if \"2026-00202\" not in tl and \"00202\" not in tl:\n continue\n candidate_texts.append((ch_id, text))\n\n if not candidate_texts:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No message referencing CASE-2026-00202 found in DM channel to @dwhitfield\"}\n\n for ch_id, text in candidate_texts:\n tl = text.lower()\n\n # Condition 1: administrative denial nature identified\n has_admin = (\n \"administrative\" in tl\n or \"prior auth\" in tl\n or \"prior authorization\" in tl\n or \"expired\" in tl\n or \"7.2.1\" in tl\n or \"no clinical\" in tl\n )\n\n # Condition 2: disputed amount ($38,400) mentioned\n has_amount = \"38,400\" in tl or \"38400\" in tl\n\n # Condition 3: authority limit cited explicitly (not just bare \"supervisor\")\n has_authority = (\n \"25,000\" in tl\n or \"25000\" in tl\n or \"coordinator authority\" in tl\n or \"authority limit\" in tl\n or \"co-sign\" in tl\n or \"cosign\" in tl\n or \"co-signature\" in tl\n or \"supervisor co\" in tl\n or \"supervisor approval\" in tl\n )\n\n # All three conditions required — no fallback\n if has_admin and has_amount and has_authority:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found qualifying
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "1d26061a-f154-497f-a6af-279a4235bb93",
|
|||
|
|
"sort_order": 5,
|
|||
|
|
"rubric_text": "In `slack_data.json`, DM channel to @dwhitfield must contain a Format B-2 escalation message referencing CASE-2026-00204 that identifies an attorney litigation threat (Webb email stating intent to seek injunctive relief and/or damages — SOP §9.2 trigger #1)",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield's user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect only DM channels that involve dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Collect messages in those DM channels referencing CASE-2026-00204\n all_relevant_messages = []\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n if \"2026-00204\" in text.lower() or \"00204\" in text.lower():\n all_relevant_messages.append((ch_id, msg))\n\n if not all_relevant_messages:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No message referencing CASE-2026-00204 found in DM channel to @dwhitfield\"}\n\n legal_keywords = [\"attorney\", \"legal\", \"litigation\", \"injunctive\", \"lawsuit\", \"court\",\n \"damages\", \"counsel\", \"webb\", \"9.2\", \"legal threat\", \"threat\"]\n\n for channel_id, msg in all_relevant_messages:\n text = msg.get(\"text\", \"\").lower()\n if any(kw in text for kw in legal_keywords):\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found B-2 escalation for CASE-2026-00204 in DM channel {channel_id} identifying attorney/legal threat\"}\n\n found_texts = [msg.get(\"text\", \"\")[:200] for _, msg in all_relevant_messages[:2]]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found messages referencing CASE-2026-00204 in DM to @dwhitfield but none identify an attorney/legal threat. Samples: {found_texts}\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "fce43237-0f7b-4041-8bf0-635dbd838634",
|
|||
|
|
"sort_order": 6,
|
|||
|
|
"rubric_text": "In `slack_data.json`, DM channel to @dwhitfield must contain a Format B-2 message referencing David Park / member VA-51104-11 that mentions both untimely filing (212 days after July 22, 2025 denial) and good cause (cardiac hospitalization July 28 – September 10, 2025), requiring supervisor authorization before any dismissal action",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield's user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect only DM channels that involve dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Collect messages in those DM channels referencing David Park / VA-51104-11\n all_relevant_messages = []\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n if \"va-51104-11\" in tl or \"va51104\" in tl or \"david park\" in tl:\n all_relevant_messages.append((ch_id, msg))\n\n if not all_relevant_messages:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No message referencing David Park / VA-51104-11 found in DM channel to @dwhitfield\"}\n\n untimely_keywords = [\"untimely\", \"212\", \"180 day\", \"late filing\", \"late appeal\",\n \"overdue\", \"past deadline\", \"out of time\"]\n good_cause_keywords = [\"good cause\", \"cardiac\", \"hospitalization\", \"hospital\"]\n\n for ch_id, msg in all_relevant_messages:\n text = msg.get(\"text\", \"\").lower()\n has_untimely = any(kw in text for kw in untimely_keywords)\n has_good_cause = any(kw in text for kw in good_cause_keywords)\n\n if has_untimely and has_good_cause:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found B-2 message for David Park/VA-51104-11 in DM channel {ch_id} \"\n f\"mentioning untimely filing and good cause (cardiac hospitalization)\"}\n\n found_texts = [msg.get(\"text\", \"\")[:300] for _, msg in all_relevant_messages[:2]]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found {len(all_relevant_messages)} message(s) referencing David Park/VA-51104-11 \"\n f\"in DM to @dwhitfield but missing both untimely filing AND good cause elements. \"\n
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "5262291b-92ba-42a8-9a1f-5a38a3ea28ce",
|
|||
|
|
"sort_order": 7,
|
|||
|
|
"rubric_text": "In `slack_data.json`, DM channel to @dwhitfield must contain a routing notification about a VA-18734-05 staff conduct grievance (SOP §10.7: grievances route to @dwhitfield, not clinical queue)",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield's user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect only DM channels that involve dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Collect messages in those DM channels that reference the grievance\n # Accept either: member ID (va-18734-05 / va18734) OR related case (2026-00203 / 00203)\n all_relevant_messages = []\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n if (\"va-18734-05\" in tl or \"va18734\" in tl or\n \"2026-00203\" in tl or \"00203\" in tl):\n all_relevant_messages.append((ch_id, msg))\n\n if not all_relevant_messages:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No message referencing VA-18734-05 or CASE-2026-00203 found in DM channel to @dwhitfield\"}\n\n grievance_keywords = [\"grievance\", \"staff conduct\", \"conduct\", \"complaint\", \"10.7\", \"routing\"]\n\n for ch_id, msg in all_relevant_messages:\n text = msg.get(\"text\", \"\").lower()\n if any(kw in text for kw in grievance_keywords):\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found VA-18734-05 staff conduct grievance routing notification \"\n f\"in DM channel {ch_id}\"}\n\n found_texts = [msg.get(\"text\", \"\")[:200] for _, msg in all_relevant_messages[:2]]\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found {len(all_relevant_messages)} message(s) referencing VA-18734-05 or \"\n f\"CASE-2026-00203 in DM to @dwhitfield but none appear to be a staff conduct \"\n f\"grievance routing notification. Samples: {found_texts}\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "57f366f7-6bdf-48f9-bca7-60283bdb1299",
|
|||
|
|
"sort_order": 8,
|
|||
|
|
"rubric_text": "In `slack_data.json`, #appeals-clinical-queue channel must contain a Format B-1 routing message referencing CASE-2026-00203 (MRI medical necessity appeal from member VA-18734-05 must be routed to clinical review per SOP §10.4 step 5)",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find appeals-clinical-queue channel\n clinical_channel_id = None\n for cid, cdata in channels.items():\n name = cdata.get(\"name\", \"\").lower()\n if \"clinical\" in name and (\"appeal\" in name or \"queue\" in name):\n clinical_channel_id = cid\n break\n elif \"clinical-queue\" in name or \"clinical_queue\" in name:\n clinical_channel_id = cid\n break\n \n if clinical_channel_id is None:\n # Search all channels\n for cid, cdata in channels.items():\n if \"clinical\" in cdata.get(\"name\", \"\").lower():\n clinical_channel_id = cid\n break\n \n if clinical_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #appeals-clinical-queue channel in slack_data.json\"}\n \n clinical_messages = messages.get(clinical_channel_id, [])\n \n for msg in clinical_messages:\n text = msg.get(\"text\", \"\").lower()\n if \"2026-00203\" in text or \"00203\" in text:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found Format B-1 routing message for CASE-2026-00203 in #appeals-clinical-queue channel (ID: {clinical_channel_id})\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No message referencing CASE-2026-00203 found in #appeals-clinical-queue channel (ID: {clinical_channel_id}). Messages found: {len(clinical_messages)}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "154a1a56-0fba-4935-baa9-7386f897eb89",
|
|||
|
|
"sort_order": 9,
|
|||
|
|
"rubric_text": "In `slack_data.json`, #appeals-clinical-queue must NOT contain any message referencing a VA-18734-05 grievance or staff conduct complaint",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find appeals-clinical-queue channel (be loose in matching)\n clinical_channel_id = None\n for cid, cdata in channels.items():\n name = cdata.get(\"name\", \"\").lower().replace(\" \", \"-\")\n # Match channels like appeals-clinical-queue, appeals_clinical_queue, etc.\n if \"clinical\" in name and (\"appeal\" in name or \"queue\" in name):\n clinical_channel_id = cid\n break\n elif \"clinical-queue\" in name or \"clinical_queue\" in name:\n clinical_channel_id = cid\n break\n \n if clinical_channel_id is None:\n for cid, cdata in channels.items():\n if \"clinical\" in cdata.get(\"name\", \"\").lower():\n clinical_channel_id = cid\n break\n \n if clinical_channel_id is None:\n # If no clinical queue channel exists at all, the prohibited messages can't be there\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No #appeals-clinical-queue channel found; prohibited messages cannot be present\"}\n \n clinical_messages = messages.get(clinical_channel_id, [])\n violations = []\n \n for msg in clinical_messages:\n text = msg.get(\"text\", \"\").lower()\n \n # Check for VA-18734-05 reference combined with grievance or staff conduct complaint\n has_case_ref = (\"va-18734-05\" in text or \"va18734-05\" in text or \"va 18734-05\" in text or \"va-1873405\" in text or \"va18734\" in text or \"18734-05\" in text)\n \n grievance_keywords = [\"grievance\", \"staff conduct\", \"conduct complaint\", \"complaint\", \"staff complaint\"]\n has_grievance = any(kw in text for kw in grievance_keywords)\n \n if has_case_ref and has_grievance:\n violations.append(f\"VA-18734-05 grievance/staff conduct complaint found in clinical queue: {msg.get('text', '')[:200]}\")\n \n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(violations)} violation(s) in #appeals-clinical-queue: {'; '.join(violations)}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"#appeals-clinical-queue does not contain any message referencing a VA-18734-05 grievance or staff conduct complaint\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "68c92946-b06f-4930-91b7-6635343d2256",
|
|||
|
|
"sort_order": 10,
|
|||
|
|
"rubric_text": "In appeals_log_master.xlsx, a new row must exist for David Park / member VA-51104-11 with Appeal Type = 'Pre-Service Standard' and Case Status = 'Intake' (Park is appealing a denied prior authorization from July 2025 — a pre-service appeal per SOP §4.3.1 — and must be logged at intake before any other action is taken per SOP §6.1, regardless of whether it will ultimately be dismissed or granted a good-cause exception)",
|
|||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n ws_path = Path(workspace_path)\n\n # Try primary target first\n target = ws_path / \"appeals_log_master.xlsx\"\n if not target.exists():\n # Broader search: any xlsx with 'appeal' in the name\n candidates = list(ws_path.rglob(\"*appeal*log*.xlsx\")) + list(ws_path.rglob(\"*appeals*.xlsx\"))\n if not candidates:\n # Even broader: any xlsx at all\n candidates = list(ws_path.rglob(\"*.xlsx\"))\n if not candidates:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"appeals_log_master.xlsx not found in workspace (no xlsx files found at all)\"}\n # Prefer files with 'appeal' in name\n appeal_files = [c for c in candidates if 'appeal' in c.name.lower()]\n target = appeal_files[0] if appeal_files else candidates[0]\n\n try:\n wb = openpyxl.load_workbook(target)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {target.name}: {e}\"}\n\n # Check all sheets, not just active\n all_found_rows = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n continue\n\n headers = [str(h).lower().strip() if h else \"\" for h in rows[0]]\n\n # Find relevant columns with loose matching\n member_id_col = None\n appeal_type_col = None\n status_col = None\n name_col = None\n\n for i, h in enumerate(headers):\n h_lower = h.lower()\n if (\"member\" in h_lower and \"id\" in h_lower) or h_lower in [\"member_id\", \"memberid\"]:\n member_id_col = i\n if \"appeal\" in h_lower and \"type\" in h_lower:\n appeal_type_col = i\n if \"status\" in h_lower:\n # Prefer 'case status' but accept any 'status'\n if \"case\" in h_lower or status_col is None:\n status_col = i\n if \"name\" in h_lower or \"member name\" in h_lower or \"appellant\" in h_lower:\n name_col = i\n\n for row in rows[1:]:\n row_vals = [str(c).strip() if c is not None else \"\" for c in row]\n row_str = \" \".join(row_vals).lower()\n\n # Check if this row is for David Park / VA-51104-11\n is_park = False\n if \"va-51104-11\" in row_str or \"va5110411\" in row_str.replace(\"-\", \"\").replace(\" \", \"\"):\n is_park = True\n if \"david\" in row_str and \"park\" in row_str:\n is_park = True\n if \"d. park\" in row_str or \"d park\" in row_str:\n is_park = True\n\n if not is_park:\n continue\n\n # Extract appeal type\n appeal_type_val = \"\"\n if appeal_type_col is not None and appeal_type_col < len(row_vals):\n appeal_type_val = row_vals[appeal_type_col].lower()\n # Also check entire row string for appeal type in case column detection failed\n appeal_type_check = appeal_type_val if appeal_type_val else row_str\n\n has_pre_service = any(t in appeal_type_check for t in\n [\"pre-service\", \"pre service\", \"preservice\", \"pre_service\"])\n # Also accept if it just says \"standard\" alongside pre-service context\n if not has_pre_service:\n has_pre_service = any(t in row_str for t in\n [\"pre-service\", \"pre service\", \"preservice\", \"pre_service\"])\n\n # Extract status\n status_val = \"\"\n if status_col is not None and status_col < len(row_vals):\n status_val =
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "987b4483-b4c1-4c7d-86e3-ee7635114574",
|
|||
|
|
"sort_order": 11,
|
|||
|
|
"rubric_text": "In `appeals_log_master.xlsx` (or equivalent spreadsheet file), a new row must exist for member VA-18734-05 with Appeal Type = 'Grievance', and that row's Related Case ID must reference CASE-2026-00203; additionally, CASE-2026-00203's row must have its Related Case ID reference the new grievance case",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n \n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not xlsx_files:\n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n \n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n \n for xlsx_path in xlsx_files:\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n except Exception as e:\n continue\n \n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows(values_only=True))\n if not rows:\n continue\n \n header_row = rows[0]\n headers = [str(h).lower().strip() if h else \"\" for h in header_row]\n \n member_id_col = next((i for i, h in enumerate(headers) if \"member\" in h and \"id\" in h), None)\n appeal_type_col = next((i for i, h in enumerate(headers) if \"appeal\" in h and \"type\" in h), None)\n related_case_col = next((i for i, h in enumerate(headers) if \"related\" in h and \"case\" in h), None)\n case_id_col = next((i for i, h in enumerate(headers) if \"case\" in h and \"id\" in h and \"related\" not in h), None)\n \n data_rows = rows[1:]\n \n # Find grievance row for VA-18734-05\n grievance_row_found = False\n grievance_case_id = None\n grievance_references_203 = False\n \n # Find CASE-2026-00203 row\n case_203_row = None\n \n for row in data_rows:\n row_str = \" \".join([str(c).lower() if c else \"\" for c in row])\n \n # Check if this is a grievance row for VA-18734-05\n has_member = \"va-18734-05\" in row_str or \"va18734\" in row_str\n is_grievance = \"grievance\" in row_str\n \n if has_member and is_grievance:\n grievance_row_found = True\n # Get case ID\n if case_id_col is not None and case_id_col < len(row):\n grievance_case_id = str(row[case_id_col]).strip() if row[case_id_col] else None\n # Check if related case references 00203\n if \"2026-00203\" in row_str or \"00203\" in row_str:\n grievance_references_203 = True\n \n # Check if this is the CASE-2026-00203 row\n if \"2026-00203\" in row_str:\n if case_id_col is not None and case_id_col < len(row):\n cell_val = str(row[case_id_col]).strip() if row[case_id_col] else \"\"\n if \"2026-00203\" in cell_val:\n case_203_row = row\n \n if not grievance_row_found:\n continue\n \n # Check cross-reference: CASE-2026-00203 must reference the grievance case\n cross_ref_ok = False\n if case_203_row is not None and grievance_case_id:\n row_203_str = \" \".join([str(c).lower() if c else \"\" for c in case_203_row])\n if grievance_case_id.lower() in row_203_str:\n cross_ref_ok = True\n elif case_203_row is not None:\n # Check if 203 row has any related case reference\n if related_case_col is not None and related_case_col < len(case_203_row):\n related_val = str(case_203_row[related_case_col]).strip() if case_203_row[related_case_col] else \"\"\n if related_val and related_val != \"none\" and related_val != \"\":\n cross_ref_ok = True\n
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776198863280",
|
|||
|
|
"sort_order": 12,
|
|||
|
|
"rubric_text": "In `slack_data.json`, the `#appeals-clinical-queue` channel must NOT contain a Format B-1 routing message referencing CASE-2026-00202.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"slack_data.json not found — no violation possible\"}\n\n with open(slack_path) as f:\n data = json.load(f)\n\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find #appeals-clinical-queue channel ID\n target_ch_id = None\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict) and \"clinical\" in ch_info.get(\"name\", \"\").lower():\n target_ch_id = ch_id\n break\n\n if target_ch_id is None:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"#appeals-clinical-queue not found — no violation possible\"}\n\n ch_messages = messages.get(target_ch_id, [])\n\n for msg in ch_messages:\n # Guard: skip if message is not a dict\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n if \"CASE-2026-00202\" in text:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Found message referencing CASE-2026-00202 in \"\n f\"#appeals-clinical-queue: '{text[:200]}'\"}\n\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"No Format B-1 routing message referencing CASE-2026-00202 \"\n \"found in #appeals-clinical-queue\"}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776198882066",
|
|||
|
|
"sort_order": 13,
|
|||
|
|
"rubric_text": "In `slack_data.json`, the `#appeals-clinical-queue` channel must NOT contain a Format B-1 routing message referencing CASE-2026-00204.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check slack_data.json for the #appeals-clinical-queue channel\n # and ensure there is NO Format B-1 routing message referencing CASE-2026-00204.\n \n # slack_data.json could be in workspace or external_services\n slack_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in workspace or external_services path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n # Find the #appeals-clinical-queue channel\n # The structure may vary; common patterns:\n # - slack_data[\"channels\"] as a list of channel objects\n # - slack_data[\"channels\"] as a dict keyed by channel name\n # - slack_data as a dict with channel names as keys\n \n channel_messages = None\n channel_name_target = \"appeals-clinical-queue\"\n \n # Try various structures\n if isinstance(slack_data, dict):\n # Check if there's a \"channels\" key\n if \"channels\" in slack_data:\n channels = slack_data[\"channels\"]\n if isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict):\n name = ch.get(\"name\", ch.get(\"channel_name\", \"\"))\n # Strip leading #\n name = name.lstrip(\"#\")\n if name == channel_name_target:\n channel_messages = ch.get(\"messages\", ch.get(\"posts\", []))\n break\n elif isinstance(channels, dict):\n for key, val in channels.items():\n clean_key = key.lstrip(\"#\")\n if clean_key == channel_name_target:\n if isinstance(val, dict):\n channel_messages = val.get(\"messages\", val.get(\"posts\", []))\n elif isinstance(val, list):\n channel_messages = val\n break\n else:\n # Maybe top-level keys are channel names\n for key, val in slack_data.items():\n clean_key = key.lstrip(\"#\")\n if clean_key == channel_name_target:\n if isinstance(val, dict):\n channel_messages = val.get(\"messages\", val.get(\"posts\", []))\n elif isinstance(val, list):\n channel_messages = val\n break\n \n if channel_messages is None:\n # If we can't find the channel at all, it might mean no messages were posted there,\n # which would satisfy the condition. But let's also do a deep text search.\n # Do a full text dump and check\n full_text = json.dumps(slack_data, indent=2).lower()\n if channel_name_target.lower() not in full_text:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Channel #appeals-clinical-queue not found in slack_data.json at all, so no violating message exists. PASS.\"}\n else:\n # Channel is referenced but we couldn't parse messages; be cautious\n # Let's do a broad text search for both B-1 and CASE-2026-00204\n has_b1_and_case = False\n if re.search(r'b[\\s-]*1', full_text) and re.search(r'case[\\s-]*2026[\\s-]*00204', full_text):\n has_b1_and_case = True\n if
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776244635360",
|
|||
|
|
"sort_order": 14,
|
|||
|
|
"rubric_text": "In slack_data.json, a DM to @dwhitfield must contain a Format B-2 escalation message referencing CASE-2026-00201 that includes: (1) Case # CASE-2026-00201, (2) Type: Pre-Service, (3) Deadline 2026-02-21, (4) Trigger: Exceeds Coordinator Authority (the $62,800 disputed amount exceeds the $25,000 coordinator limit per SOP §9.1.1), and (5) a supervisor ask (approval, guidance, or direct takeover per SOP §9.3 step 2).",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Identify dwhitfield user ID\n dwhitfield_id = None\n for uid, udata in users.items():\n name = udata.get(\"name\", \"\").lower()\n display = udata.get(\"display_name\", \"\").lower()\n real = udata.get(\"real_name\", \"\").lower()\n if any(\"whitfield\" in s or \"dwhitfield\" in s for s in (name, display, real)):\n dwhitfield_id = uid\n break\n\n # Collect DM channels involving dwhitfield\n dm_channel_ids = []\n for ch_id, ch_info in channels.items():\n if not isinstance(ch_info, dict):\n continue\n is_channel = ch_info.get(\"is_channel\", True)\n ch_name = ch_info.get(\"name\", \"\").lower()\n members = ch_info.get(\"members\", [])\n\n is_dm = (\n not is_channel\n or ch_info.get(\"is_im\", False)\n or ch_info.get(\"is_mpim\", False)\n or ch_name.startswith(\"dm\")\n )\n if not is_dm:\n continue\n\n involves_dwhitfield = (\n \"dwhitfield\" in ch_name\n or \"whitfield\" in ch_name\n or (dwhitfield_id and dwhitfield_id in members)\n )\n if involves_dwhitfield:\n dm_channel_ids.append(ch_id)\n\n if not dm_channel_ids:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No DM channel to @dwhitfield found in slack_data.json\"}\n\n # Search DM messages for the five required elements\n best_score = 0\n best_excerpt = \"\"\n best_missing = []\n\n for ch_id in dm_channel_ids:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n\n # Element 1: Case ID\n has_case = bool(re.search(r\"case[\\-\\s_]*2026[\\-\\s_]*00201\", tl))\n\n # Element 2: Pre-Service type\n has_preservice = bool(re.search(r\"pre[\\-\\s_]?service\", tl))\n\n # Element 3: Deadline 2026-02-21\n has_deadline = bool(re.search(\n r\"2026[\\-/]02[\\-/]21|02[\\-/]21[\\-/]2026|feb(?:ruary)?\\s*21|21\\s*feb|2/21\", tl\n ))\n\n # Element 4: Escalation trigger — Exceeds Coordinator Authority\n has_trigger = (\n \"exceeds coordinator authority\" in tl\n or \"exceed\" in tl and \"coordinator\" in tl\n or \"exceed\" in tl and \"authority\" in tl\n or (\"62,800\" in tl or \"62800\" in tl) and (\"25,000\" in tl or \"25000\" in tl)\n or \"exceeds coordinator\" in tl\n )\n\n # Element 5: Supervisor ask — one of: approval, guidance, direct takeover\n # (Fix: previous version used \\x08 backspace bytes instead of \\b word boundaries.\n # Substring check is robust and matches the SOP vocabulary exactly.)\n has_supervisor_ask = any(t in tl for t in (\n \"approval\", \"guidance\", \"direct takeover\", \"takeover\",\n
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776254584501",
|
|||
|
|
"sort_order": 15,
|
|||
|
|
"rubric_text": "In `mailbox.json` (Sent folder), an acknowledgment email addressed to `m.soto.personal@gmail.com` must reference CASE-2026-00202 and state a decision deadline of 2026-04-18.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check the mailbox.json for a sent email matching the criteria.\n # The mailbox could be in the workspace or in external_services_path.\n \n mailbox_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"mailbox.json\")\n candidates.append(Path(workspace_path) / \"mailbox.json\")\n \n for c in candidates:\n if c.exists():\n mailbox_path = c\n break\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in workspace or external_services_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n \n # Filter for Sent folder\n sent_emails = [e for e in emails if str(e.get(\"folder\", \"\")).lower().strip() in (\"sent\", \"sent items\", \"sent mail\")]\n \n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in the Sent folder of mailbox.json.\"}\n \n # Now look for an email to m.soto.personal@gmail.com that references CASE-2026-00202 and deadline 2026-04-18\n target_addr = \"m.soto.personal@gmail.com\"\n case_ref = \"case-2026-00202\"\n # Various representations of the deadline date\n deadline_patterns = [\n \"2026-04-18\", \"04/18/2026\", \"4/18/2026\",\n \"april 18, 2026\", \"april 18 2026\", \"18 april 2026\",\n \"apr 18, 2026\", \"apr 18 2026\", \"2026/04/18\",\n \"04-18-2026\", \"4-18-2026\",\n \"18/04/2026\", \"18-04-2026\"\n ]\n \n matching_emails = []\n \n for email in sent_emails:\n to_addr = str(email.get(\"to_addr\", \"\")).lower().strip()\n # Also check 'to' field in case of different key naming\n to_addr2 = str(email.get(\"to\", \"\")).lower().strip()\n combined_to = to_addr + \" \" + to_addr2\n # Check if target address appears in to_addr (could be comma-separated list)\n if target_addr not in combined_to:\n continue\n \n # Combine subject and body for searching\n subject = str(email.get(\"subject\", \"\"))\n body = str(email.get(\"body_text\", \"\"))\n # Also check 'body' key\n body2 = str(email.get(\"body\", \"\"))\n full_text = (subject + \" \" + body + \" \" + body2).lower()\n \n # Check for case reference - be loose with separators\n # Normalize: remove dashes and spaces to check\n normalized_text = full_text.replace(\"-\", \"\").replace(\" \", \"\")\n has_case = case_ref.replace(\"-\", \"\") in normalized_text or case_ref in full_text\n \n # Check for deadline\n has_deadline = any(dp in full_text for dp in deadline_patterns)\n \n if has_case and has_deadline:\n matching_emails.append(email)\n \n if matching_emails:\n e = matching_emails[0]\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found matching sent email to {e.get('to_addr', e.get('to', ''))} with subject '{e.get('subject', '')}'. It references CASE-2026-00202 and includes a decision deadline of 2026-04-18.\"\n }\n \n # Provide debugging info\n sent_to_soto = [e for e in sent_emails if target_addr in (str(e.get('to_addr', '')).lower() + ' ' + str(e.get('to', '')).lower())]\n if sent_to_soto:\n details = []\n for e in sent_to_soto:\n full = (str(e.get('subject','')) + ' ' + str(e.get('body_text','')) + ' ' + str(e.get('body','')))
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776254974071",
|
|||
|
|
"sort_order": 16,
|
|||
|
|
"rubric_text": "In `mailbox.json` (Sent folder), an acknowledgment email addressed to `h.reyes.hector@gmail.com` must reference `CASE-2026-00203` and state a decision deadline of `2026-04-18`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # First try external_services_path for mailbox.json, then workspace_path\n mailbox_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"mailbox.json\")\n candidates.append(Path(workspace_path) / \"mailbox.json\")\n \n for c in candidates:\n if c.exists():\n mailbox_path = c\n break\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in workspace or external_services_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n \n # Filter for Sent folder emails\n sent_emails = [e for e in emails if str(e.get(\"folder\", \"\")).lower() in (\"sent\", \"sent items\", \"sent mail\")]\n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in the Sent folder.\"}\n \n target_addr = \"h.reyes.hector@gmail.com\"\n case_ref = \"CASE-2026-00203\"\n # Accept various date representations for 2026-04-18\n deadline_patterns = [\n r\"2026[\\-/]04[\\-/]18\",\n r\"04[\\-/]18[\\-/]2026\",\n r\"18[\\-/]04[\\-/]2026\",\n r\"April\\s+18[,\\s]+2026\",\n r\"18\\s+April[,\\s]+2026\",\n r\"Apr\\.?\\s+18[,\\s]+2026\",\n r\"18\\s+Apr\\.?[,\\s]+2026\",\n ]\n \n matching_emails = []\n for email in sent_emails:\n to_addr = str(email.get(\"to_addr\", \"\")).lower().strip()\n # Check if target address appears in to_addr (could be comma-separated list)\n if target_addr.lower() not in to_addr:\n continue\n \n # Combine subject and body for content search\n subject = str(email.get(\"subject\", \"\"))\n body = str(email.get(\"body_text\", \"\"))\n full_text = subject + \" \" + body\n \n # Check for case reference (case-insensitive, flexible on separator)\n case_pattern = re.compile(r\"CASE[\\-\\s]?2026[\\-\\s]?00203\", re.IGNORECASE)\n has_case = bool(case_pattern.search(full_text))\n \n # Check for deadline date\n has_deadline = False\n for dp in deadline_patterns:\n if re.search(dp, full_text, re.IGNORECASE):\n has_deadline = True\n break\n \n if has_case and has_deadline:\n matching_emails.append({\n \"subject\": subject,\n \"to\": to_addr,\n \"has_case\": has_case,\n \"has_deadline\": has_deadline\n })\n \n if matching_emails:\n e = matching_emails[0]\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found sent email to {target_addr} with subject '{e['subject']}' referencing {case_ref} and containing decision deadline 2026-04-18.\"\n }\n \n # Provide diagnostic info on what was found\n to_target = [e for e in sent_emails if target_addr.lower() in str(e.get(\"to_addr\", \"\")).lower()]\n if not to_target:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No sent emails found addressed to {target_addr}. Sent emails found to: {[e.get('to_addr','') for e in sent_emails]}\"\n }\n \n # Emails to target exist but missing case or deadline\n diag = []\n for email in to_target:\n subject = str(email.get(\"subject\", \"\"))\n body = str(email.get(\"body_text\", \"\"))\n full_text = subject + \" \" + body\n case_pattern
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776255060748",
|
|||
|
|
"sort_order": 17,
|
|||
|
|
"rubric_text": "In `mailbox.json` (Sent folder), an acknowledgment email addressed to `d.park.personal@gmail.com` must reference David Park in the body.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n mailbox_path = None\n\n # Try external_services_path first\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path or workspace_path.\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n\n target_addr = \"d.park.personal@gmail.com\"\n # Pattern to find \"David Park\" in the body (case-insensitive, allow slight variations)\n david_park_pattern = re.compile(r'David\\s+Park', re.IGNORECASE)\n\n sent_to_target = []\n matching_emails = []\n\n for email in emails:\n folder = (email.get(\"folder\") or \"\").strip()\n if folder.lower() != \"sent\":\n continue\n\n to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").strip().lower()\n # Be generous: check if target address appears anywhere in to_addr field\n if target_addr.lower() not in to_addr:\n continue\n\n sent_to_target.append(email)\n\n # Check the body for \"David Park\"\n body = email.get(\"body_text\") or email.get(\"body\") or \"\"\n subject = email.get(\"subject\") or \"\"\n # Check body primarily, but also be generous and check subject\n combined = body + \" \" + subject\n\n if david_park_pattern.search(combined):\n matching_emails.append(email)\n\n if matching_emails:\n e = matching_emails[0]\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found sent acknowledgment email to {target_addr} referencing David Park. Subject: \\\"{e.get('subject', '')}\\\". Found {len(matching_emails)} matching email(s).\"\n }\n\n if sent_to_target:\n subjects = [e.get(\"subject\", \"(no subject)\") for e in sent_to_target]\n bodies_preview = [(e.get(\"body_text\") or e.get(\"body\") or \"\")[:200] for e in sent_to_target]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(sent_to_target)} sent email(s) to {target_addr}, but none reference 'David Park' in the body. Subjects: {subjects}. Body previews: {bodies_preview}\"\n }\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No sent emails found addressed to {target_addr}. Total emails in mailbox: {len(emails)}.\"\n }\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776255832960",
|
|||
|
|
"sort_order": 18,
|
|||
|
|
"rubric_text": "In `slack_data.json`, the `#appeals-clinical-queue` channel must contain a Format B-1 routing message containing Case # `CASE-2026-00196`, Type `Pre-service Standard`, Standard of Review `Medical Necessity`, Deadline `2026-03-15`, and Parity flag `No`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n for base in [external_services_path, workspace_path]:\n if base is None:\n continue\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n break\n\n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Try to find #appeals-clinical-queue by channel name\n clinical_channel_id = None\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict) and \"clinical\" in ch_info.get(\"name\", \"\").lower():\n clinical_channel_id = ch_id\n break\n\n # If channel metadata absent, search all message channels for clinical-queue content\n channels_to_search = []\n if clinical_channel_id:\n channels_to_search = [clinical_channel_id]\n else:\n channels_to_search = list(messages.keys())\n\n for ch_id in channels_to_search:\n for msg in messages.get(ch_id, []):\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n if not isinstance(text, str):\n continue\n tl = text.lower()\n if (\"2026-00196\" in tl or \"00196\" in tl) and \"pre-service\" in tl and \"medical\" in tl and \"2026-03-15\" in tl:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Found Format B-1 routing message for CASE-2026-00196 in channel {ch_id}\"}\n\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No Format B-1 routing message for CASE-2026-00196 found in slack_data.json\"}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
}
|
|||
|
|
]
|