Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,86 @@
[
{
"id": "847ec739-0f1c-4b09-baea-e72a664b9d57",
"sort_order": 0,
"rubric_text": "In `jira_state.json`, a new Jira ticket (beyond pre-existing issues MCA_APPEALS-1 through MCA_APPEALS-5) must exist with the summary or description text containing at least 3 of the following terms: MBR-2026-04471, 88500, parity, Dr. Ontario, concurrent care. Note: the MCP tool assigns the key based on project issue count, so the key may be MCA-APPEALS-6, MCA-6, CASE-1, or similar — the exact key is not checked.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n # Pre-existing issues that were present before the agent started\n pre_existing_keys = {\n \"MCA_APPEALS-1\", \"MCA_APPEALS-2\", \"MCA_APPEALS-3\",\n \"MCA_APPEALS-4\", \"MCA_APPEALS-5\"\n }\n\n issues = data.get(\"issues\", {})\n\n # Find any new issue (not in the pre-existing set) — the MCP tool assigns\n # the key based on project issue count, so it may be MCA-APPEALS-6, MCA-6,\n # CASE-1, or similar depending on which project key the agent used.\n new_issues = {k: v for k, v in issues.items() if k not in pre_existing_keys}\n\n if not new_issues:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": (\n \"No new Jira ticket found beyond pre-existing issues \"\n \"(MCA_APPEALS-1 through 90). A new Jira ticket must be \"\n \"created with required case information.\"\n )\n }\n\n # Take the first (should be only one) new issue found\n ticket_key, ticket = next(iter(new_issues.items()))\n\n # Collect all text from the ticket recursively\n def collect_text(obj):\n texts = []\n if isinstance(obj, str):\n texts.append(obj)\n elif isinstance(obj, dict):\n for v in obj.values():\n texts.extend(collect_text(v))\n elif isinstance(obj, list):\n for item in obj:\n texts.extend(collect_text(item))\n elif obj is not None:\n texts.append(str(obj))\n return texts\n\n all_text = \" \".join(collect_text(ticket)).lower()\n\n # Check for required terms (case-insensitive)\n # Rubric: at least 3 of these 5 terms\n required_terms = {\n \"MBR-2026-04471\": [\"mbr-2026-04471\", \"mbr202604471\", \"mbr 2026 04471\", \"04471\"],\n \"88500\": [\"88500\", \"88,500\", \"88500.00\", \"$88,500\", \"$88500\"],\n \"parity\": [\"parity\", \"mhpaea\"],\n \"Dr. Ontario\": [\"ontario\"],\n \"concurrent care\": [\"concurrent\"],\n }\n\n found_terms = []\n missing_terms = []\n for term_label, variants in required_terms.items():\n term_found = any(variant in all_text for variant in variants)\n if term_found:\n found_terms.append(term_label)\n else:\n missing_terms.append(term_label)\n\n # Rubric requires at least 3 of 5 terms\n min_required = 3\n total_terms = len(required_terms)\n score = len(found_terms) / total_terms\n\n if len(found_terms) >= min_required:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": (\n f\"New Jira ticket ({ticket_key}) found with {len(found_terms)}/{total_terms} \"\n f\"required terms present: {', '.join(found_terms)}.\"\n + (f\" Missing: {', '.join(missing_terms)}.\" if missing_terms else \"\")\n )\n }\n else:\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": (\n f\"New Jira ticket ({ticket_key}) found but only {len(found_terms)}/{min_required} \"\n f\"required terms present (need at least {min_required}). \"\n f\"Found: {', '.join(found_terms) if found_terms else 'none'}. \"\n f\"Missing: {', '.join(missing_terms)}.\"\n )\n }\n",
"criterion_type": "expected_output"
},
{
"id": "986fba7a-6a01-4681-bb62-af638f93c172",
"sort_order": 1,
"rubric_text": "In `slack_data.json`, a DM must exist to @parity-committee containing Format B-3 fields: Case # MCA-APPEALS-91, Service Classification, Comparable M/S Classification, Flag Indicator, Case Folder, and Decision Deadline of 2026-04-16 12:00 PM PT. This notification must be a DM (not a public channel post in #appeals-parity).",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find parity-committee user IDs using various name fields\n parity_committee_user_ids = set()\n for uid, uinfo in users.items():\n name = uinfo.get(\"name\", \"\").lower()\n real_name = uinfo.get(\"real_name\", \"\").lower()\n display = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n if \"parity\" in name or \"parity\" in real_name or \"parity\" in display or \"parity-committee\" in name:\n parity_committee_user_ids.add(uid)\n\n # Identify DM channels to parity-committee\n dm_channel_ids_to_parity = set()\n for channel_id, cinfo in channels.items():\n is_im = cinfo.get(\"is_im\", False)\n is_mpim = cinfo.get(\"is_mpim\", False)\n is_dm = is_im or is_mpim or str(channel_id).startswith(\"D\")\n if is_dm:\n dm_user = cinfo.get(\"user\", \"\")\n if dm_user in parity_committee_user_ids:\n dm_channel_ids_to_parity.add(channel_id)\n ch_name = cinfo.get(\"name\", \"\").lower()\n if \"parity\" in ch_name:\n dm_channel_ids_to_parity.add(channel_id)\n\n found_b3 = False\n found_msg = None\n best_missing = None\n\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n\n channel_info = channels.get(channel_id, {})\n channel_name = channel_info.get(\"name\", \"\").lower()\n is_im = channel_info.get(\"is_im\", False)\n is_mpim = channel_info.get(\"is_mpim\", False)\n is_dm = is_im or is_mpim or str(channel_id).startswith(\"D\")\n\n # Skip public #appeals-parity channel - rubric says must be DM not public channel\n is_public_appeals_parity = (\"appeals-parity\" in channel_name) and not is_dm\n if is_public_appeals_parity:\n continue\n\n is_parity_dm_channel = channel_id in dm_channel_ids_to_parity\n dm_user = channel_info.get(\"user\", \"\")\n if dm_user in parity_committee_user_ids:\n is_parity_dm_channel = True\n if \"parity\" in channel_name and is_dm:\n is_parity_dm_channel = True\n\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n\n # Must reference MCA-APPEALS-91 (the rubric case number)\n # Accept various formats: MCA-APPEALS-91, MCA-APPEALS-0091, CASE-2026-00091, etc.\n has_case_ref = False\n case_patterns = [\n \"mca-appeals-91\", \"mca-appeals-091\", \"mca-appeals-0091\",\n \"mca appeals 91\", \"mca appeals 091\", \"mca appeals 0091\",\n \"case-2026-00091\", \"case-2026-0091\", \"case-2026-091\", \"case-2026-91\",\n \"case 2026 00091\",\n \"appeals-91\", \"appeals-091\", \"appeals-0091\"\n ]\n for pat in case_patterns:\n if pat in text_lower:\n has_case_ref = True\n break\n # Also try regex for variations\n if not has_case_ref:\n if re.search(r'mca[\\-\\s]?appeals[\\-\\s]?0*91\\b', text_lower):\n has_case_ref = True\n if re.search(r'case[\\-\\s]?2026[\\-\\s]?0*91\\b', text_lower):\n has_case_ref = True\n\n if not has_case_ref:\n continue\n\n # Check if directed to parity-committee\n mentions_parity = (\"parity-committee\" in text_lower or\n \"parity_committee\" in text_lower or\n \"parity committee\" in text_lower or\n \"@parity\" in text_lower or\n any(uid in text for uid in parity_committee_user_ids))\n\n directed_to_parity = is_parity_dm_channel or mentions_parity\n\n if not directed_to_parity:\n continue\n\n # Check for B-3 required fields\n current_missing = []\n\n # 1. Case # MCA-APPEALS-91 - already confirmed above\n\n # 2. Service Classification\n has_service_class = any(term in text_lower for term in [\n \"service cLine truncated
"criterion_type": "expected_output"
},
{
"id": "104af031-8aec-49dd-8e46-ef40afa79a2a",
"sort_order": 2,
"rubric_text": "In `slack_data.json`, a DM must exist to @aontario referencing MCA-APPEALS-91 and requesting parity clinical routing. Use cinfo.get(\"user\", \"\") rather than cinfo.get(\"members\", []). DM channels store the other party's user ID in user, not members.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find aontario user id(s)\n aontario_ids = set()\n for uid, uinfo in users.items():\n name = uinfo.get(\"name\", \"\").lower()\n real_name = uinfo.get(\"real_name\", \"\").lower()\n display = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n email = uinfo.get(\"profile\", {}).get(\"email\", \"\").lower()\n if \"aontario\" in name or \"aontario\" in display or \"ontario\" in real_name or \"aontario\" in email:\n aontario_ids.add(uid)\n\n # Find DM channels involving aontario\n # Per rubric: use cinfo.get(\"user\", \"\") rather than cinfo.get(\"members\", []).\n # DM channels store the other party's user ID in \"user\", not \"members\".\n aontario_channels = set()\n for cid, cinfo in channels.items():\n is_dm = cinfo.get(\"is_im\", False) or cinfo.get(\"is_mpim\", False) or str(cid).startswith(\"D\")\n cname = cinfo.get(\"name\", \"\").lower()\n\n # Primary: use \"user\" field for DM channels (per rubric)\n dm_user = cinfo.get(\"user\", \"\")\n if is_dm and dm_user in aontario_ids:\n aontario_channels.add(cid)\n\n # Fallback: check members for mpim or other formats\n members = cinfo.get(\"members\", [])\n if is_dm and any(uid in members for uid in aontario_ids):\n aontario_channels.add(cid)\n\n # Also match by channel name\n if \"aontario\" in cname or \"ontario\" in cname:\n aontario_channels.add(cid)\n\n # Build a pattern to match various forms of the appeal reference:\n # MCA-APPEALS-91, CASE-2026-00091, case 91, appeal 91, etc.\n appeal_patterns = [\n r\"mca[\\-\\s]?appeals[\\-\\s]?0*91\",\n r\"case[\\-\\s]?2026[\\-\\s]?0*91\",\n r\"appeal[s]?[\\-\\s#:]*0*91\",\n r\"\\b0*91\\b\"\n ]\n\n def text_references_appeal(text):\n t = text.lower()\n # Direct string checks\n if \"mca-appeals-91\" in t or \"mca-appeals-091\" in t:\n return True\n if \"case-2026-00091\" in t or \"case-2026-0091\" in t or \"case-2026-091\" in t or \"case-2026-91\" in t:\n return True\n for pat in appeal_patterns:\n if re.search(pat, t):\n return True\n return False\n\n def text_has_parity_clinical_routing(text):\n t = text.lower()\n # Check for parity clinical routing - be loose\n has_parity = \"parity\" in t\n has_clinical = \"clinical\" in t\n has_routing = \"routing\" in t or \"route\" in t or \"review\" in t or \"assign\" in t or \"forward\" in t or \"refer\" in t\n # Accept: parity + clinical, parity + routing, clinical + routing, or any two\n if has_parity and (has_clinical or has_routing):\n return True\n if has_clinical and has_routing:\n return True\n # Also accept just \"parity\" mentioned alongside appeal reference (loose)\n if has_parity:\n return True\n # Accept clinical routing without parity as well (loose)\n if has_clinical and has_routing:\n return True\n return False\n\n found = False\n found_msg = None\n debug_candidates = []\n\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n channel_info = channels.get(channel_id, {})\n is_dm = channel_info.get(\"is_im\", False) or channel_info.get(\"is_mpim\", False) or str(channel_id).startswith(\"D\")\n\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n\n if not text_references_appeal(text):\n continue\n\n # Check if this message is directed to aontario\n is_aontario_channel = channel_id in aontario_channels\n mentions_aontario = \"aontario\" in text_lower or \"ontario\" in text_lower or any(uid in text for uid in aontario_ids)\n\n is_to_aontario = is_aontario_channel or mentions_aontario\n\n if not is_to_aontario:\n debug_candidates.append(f\"channel={channel_id}, is_dm={is_dm}, text_preview={text[:100]}\")\n continue\n\n # CheLine truncated
"criterion_type": "expected_output"
},
{
"id": "c5731f16-8ccc-49ca-9334-988b66fb349a",
"sort_order": 3,
"rubric_text": "In `slack_data.json`, a DM must exist to @dwhitfield containing Format B-2 escalation with: Case # MCA-APPEALS-91, Type = Concurrent, Deadline = 2026-04-16 12:00 PM PT, and all three triggers: 'Exceeds Coordinator Authority' ($88,500 > $25K), parity flag raised, and concurrent care termination within 72 hours. Use cinfo.get(\"user\", \"\") rather than cinfo.get(\"members\", []). DM channels store the other party's user ID in user, not members. Check it is a DM, do not use cinfo.get(\"members\", []), I repeat do not use this, instead use cinfo.get(\"user\", \"\") in the code.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find dwhitfield user IDs\n dwhitfield_ids = set()\n for uid, uinfo in users.items():\n name = uinfo.get(\"name\", \"\").lower()\n display = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n real_name = uinfo.get(\"real_name\", \"\").lower()\n if \"dwhitfield\" in name or \"whitfield\" in real_name or \"dwhitfield\" in display:\n dwhitfield_ids.add(uid)\n\n # Find DM channels involving dwhitfield using cinfo.get(\"user\", \"\") per rubric\n dwhitfield_dm_channels = set()\n for cid, cinfo in channels.items():\n is_dm = cinfo.get(\"is_im\", False) or cinfo.get(\"is_mpim\", False) or str(cid).startswith(\"D\")\n if is_dm:\n dm_user = cinfo.get(\"user\", \"\")\n if dm_user in dwhitfield_ids:\n dwhitfield_dm_channels.add(cid)\n\n found = False\n found_msg = None\n missing_fields = []\n best_missing = None\n\n # Normalize the case number patterns we accept\n # Rubric says: Case # MCA-APPEALS-91\n # Accept various representations: MCA-APPEALS-91, CASE-2026-00091, case 91, etc.\n def has_case_ref(text_lower, text):\n if \"mca-appeals-91\" in text_lower:\n return True\n if \"mca-appeals-0091\" in text_lower:\n return True\n if \"mca-appeals-00091\" in text_lower:\n return True\n if \"case-2026-00091\" in text_lower:\n return True\n if \"case-2026-0091\" in text_lower:\n return True\n # Flexible: any mention of appeals and 91 together\n if \"appeals\" in text_lower and (\"91\" in text or \"-91\" in text):\n return True\n return False\n\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n\n channel_info = channels.get(channel_id, {})\n is_dm = channel_info.get(\"is_im\", False) or channel_info.get(\"is_mpim\", False) or str(channel_id).startswith(\"D\")\n\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n\n if not has_case_ref(text_lower, text):\n continue\n\n # Must be a DM to dwhitfield or mention dwhitfield\n is_dwhitfield_channel = channel_id in dwhitfield_dm_channels\n mentions_dwhitfield = \"dwhitfield\" in text_lower or \"whitfield\" in text_lower or any(uid in text for uid in dwhitfield_ids)\n\n # Accept if: known DM channel to dwhitfield, OR it's a DM that mentions dwhitfield,\n # OR it's any channel that mentions dwhitfield (be lenient)\n if not (is_dwhitfield_channel or mentions_dwhitfield):\n continue\n\n # Check B-2 fields\n missing = []\n\n # Type: Concurrent\n if \"concurrent\" not in text_lower:\n missing.append(\"Type=Concurrent\")\n\n # Deadline 2026-04-16 12:00 PM PT (lenient: just need 2026-04-16 or April 16)\n if \"2026-04-16\" not in text and \"april 16\" not in text_lower and \"04/16/2026\" not in text and \"4/16/2026\" not in text and \"04-16-2026\" not in text and \"apr 16\" not in text_lower:\n missing.append(\"Deadline 2026-04-16\")\n\n # Trigger 1: Exceeds coordinator authority / $25K / $88,500\n has_exceed = (\"exceed\" in text_lower or \"coordinator authority\" in text_lower or\n \"25,000\" in text or \"25000\" in text or \"25k\" in text_lower or\n \"88,500\" in text or \"88500\" in text or \"$88\" in text or\n \"authority\" in text_lower)\n if not has_exceed:\n missing.append(\"Trigger: Exceeds Coordinator Authority\")\n\n # Trigger 2: Parity flag\n has_parity = \"parity\" in text_lower\n if not has_parity:\n missing.append(\"Trigger: Parity flag\")\n\n # Trigger 3: Concurrent care termination within 72 hours\n has_concurrent_72 = (\"72\" in text and (\"concurrent\" in text_lower or \"termination\" in text_lower or \"hour\" in text_lower)) or \\\n Line truncated
"criterion_type": "expected_output"
},
{
"id": "9d0cb234-202f-4f1c-8cad-e01a15904e8d",
"sort_order": 4,
"rubric_text": "In `mailbox.json`, a sent email must exist to Desert Springs Recovery Center (the triggering sender) referencing MCA-APPEALS-91, using Template A-1 with expedited header block containing exactly: 'Expedited Basis: Treating Physician Certification' and 'Decision Due By: 2026-04-16 12:00 PM PT'.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n mail_path = Path(external_services_path) / \"mailbox.json\"\n if not mail_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mail_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() in [\"sent\", \"sent items\", \"sent mail\"]]\n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent emails found in mailbox.json\"}\n\n # We need to find a sent email that:\n # 1. Is addressed to Desert Springs Recovery Center\n # 2. References MCA-APPEALS-91 (which maps to CASE-2026-00091 or similar)\n # 3. Uses Template A-1\n # 4. Contains expedited header block with:\n # - 'Expedited Basis: Treating Physician Certification'\n # - 'Decision Due By: 2026-04-16 12:00 PM PT'\n\n best_match = None\n best_missing = None\n\n for email in sent_emails:\n body_full = email.get(\"body_text\", \"\") or \"\"\n body_lower = body_full.lower()\n to_addr = (email.get(\"to_addr\", \"\") or \"\").lower()\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n combined_lower = body_lower + \" \" + subject + \" \" + to_addr\n\n # Check reference to MCA-APPEALS-91 (accept various formats)\n has_case_ref = False\n case_patterns = [\n \"mca-appeals-91\", \"mca-appeals-091\", \"mca-appeals-0091\",\n \"case-2026-00091\", \"case-2026-0091\", \"case-2026-091\", \"case-2026-91\",\n \"appeals-91\", \"appeals-091\", \"appeals-0091\", \"appeals-00091\",\n \"91\"\n ]\n # Be generous: check for any reasonable reference\n for pat in case_patterns[:-1]: # skip bare \"91\" unless nothing else matches\n if pat in combined_lower:\n has_case_ref = True\n break\n if not has_case_ref:\n # Check with regex for appeal/case number 91\n if re.search(r'(mca|case|appeal)[^a-z0-9]*0*91', combined_lower):\n has_case_ref = True\n if not has_case_ref:\n continue\n\n # Check Desert Springs Recovery Center as recipient\n is_to_desert_springs = False\n if \"desert\" in to_addr and \"spring\" in to_addr:\n is_to_desert_springs = True\n elif \"desertspring\" in to_addr:\n is_to_desert_springs = True\n elif \"desert springs\" in combined_lower:\n is_to_desert_springs = True\n elif \"desert\" in to_addr:\n is_to_desert_springs = True # generous\n if not is_to_desert_springs:\n continue\n\n # Now check the specific required content\n missing = []\n\n # Check for expedited basis: treating physician certification\n if \"treating physician certification\" in body_lower:\n pass # exact match\n elif \"treating physician\" in body_lower and \"certification\" in body_lower:\n pass # close enough\n elif \"treating physician\" in body_lower:\n pass # lenient\n else:\n missing.append(\"Expedited Basis: Treating Physician Certification\")\n\n # Check for Decision Due By: 2026-04-16 12:00 PM PT\n has_date = (\"2026-04-16\" in body_full or \"04/16/2026\" in body_full or \n \"april 16\" in body_lower or \"04-16-2026\" in body_full or\n \"4/16/2026\" in body_full or \"april 16, 2026\" in body_lower)\n if not has_date:\n missing.append(\"Decision Due By date (2026-04-16)\")\n\n has_time = (\"12:00 pm\" in body_lower or \"12:00pm\" in body_lower or \n \"12 pm\" in body_lower or \"12pm\" in body_lower or\n \"noon\" in body_lower or \"12:00 PM\" in body_full or\n \"12:00\" in body_full)\n if not has_time:\n missing.append(\"Decision Due By time (12:00 PM)\")\n\n # Check for expedited keyword\n if \"expedited\" not in body_lower:\n missing.append(\"Expedited header/keyword\")\n\n # Check for Template A-1 reference (lenient - could be in subject or body)\n has_template = (\"a-1\" in combined_lower or \"template a\" in combined_lower or\n \"a1\" in combined_lower or \"template\" in combined_lower)\n if not has_template:\n # Don't require explicit template mention if the content matches the template format\n pass # Be lenient - the template content matters more than naming it\n\n # Allow up to Line truncated
"criterion_type": "expected_output"
},
{
"id": "fa4a3f2f-ed67-452a-affd-07783e53daa3",
"sort_order": 5,
"rubric_text": "In `slack_data.json`, no Format B-3 parity notification must be posted in any public channel named #appeals-parity (the correct destination is a DM to @parity-committee only, per SOP s.8.3).",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n # Find #appeals-parity channel\n parity_channel_ids = set()\n for cid, cinfo in channels.items():\n cname = cinfo.get(\"name\", \"\").lower()\n is_dm = cinfo.get(\"is_im\", False) or cinfo.get(\"is_mpim\", False)\n if \"appeals-parity\" in cname and not is_dm:\n parity_channel_ids.add(cid)\n if not parity_channel_ids:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No #appeals-parity channel found; no erroneous public parity posts possible\"}\n for channel_id in parity_channel_ids:\n msgs = messages.get(channel_id, [])\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\").lower()\n if \"case-2026-00091\" in text:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found B-3 parity notification posted in public #appeals-parity channel (should be DM only): '{msg.get('text','')[:200]}'\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No Format B-3 parity notification found in public #appeals-parity channel; correctly routed as DM only\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "1d4b39bd-d917-40e4-b7ff-875ea8d42055",
"sort_order": 6,
"rubric_text": "In `slack_data.json`, no clinical routing messages for CASE-2026-00091 must be sent to @malvarado or @hbergstrom, and no Format B-1 must be posted in #appeals-clinical-queue for CASE-2026-00091 (parity flag overrides standard clinical routing per SOP s.8.3).",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n users = data.get(\"users\", {})\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n # Find malvarado and hbergstrom user IDs\n bad_reviewer_ids = set()\n bad_reviewer_names = [\"malvarado\", \"hbergstrom\", \"alvarado\", \"bergstrom\"]\n for uid, uinfo in users.items():\n name = uinfo.get(\"name\", \"\").lower()\n real_name = uinfo.get(\"real_name\", \"\").lower()\n display = uinfo.get(\"profile\", {}).get(\"display_name\", \"\").lower()\n for bad_name in bad_reviewer_names:\n if bad_name in name or bad_name in real_name or bad_name in display:\n bad_reviewer_ids.add(uid)\n # Find #appeals-clinical-queue channel\n clinical_channel_ids = set()\n for cid, cinfo in channels.items():\n cname = cinfo.get(\"name\", \"\").lower()\n if \"clinical-queue\" in cname or \"clinical_queue\" in cname:\n clinical_channel_ids.add(cid)\n violations = []\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\")\n text_lower = text.lower()\n if \"case-2026-00091\" not in text_lower:\n continue\n # Check for malvarado/hbergstrom mentions\n for bad_name in bad_reviewer_names:\n if bad_name in text_lower:\n violations.append(f\"Message to/mentioning {bad_name} references CASE-2026-00091: '{text[:150]}'\")\n # Check for bad reviewer user IDs\n for uid in bad_reviewer_ids:\n if uid in text:\n violations.append(f\"Message mentions bad reviewer UID {uid} for CASE-2026-00091\")\n # Check for B-1 in clinical queue\n if channel_id in clinical_channel_ids:\n violations.append(f\"Found message in #appeals-clinical-queue referencing CASE-2026-00091 (should not be routed via B-1): '{text[:150]}'\")\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Incorrect clinical routing violations found: \" + \"; \".join(violations)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No routing to @malvarado or @hbergstrom, no B-1 in #appeals-clinical-queue for CASE-2026-00091\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "0977af41-e16d-4dee-854a-3ec1f2afd70d",
"sort_order": 7,
"rubric_text": "In `slack_data.json`, no public Slack channel post must contain the member ID 'MBR-2026-04471' or other PHI for this case (PHI must not appear in any public channel for CASE-2026-00091).",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n phi_patterns = [\"mbr-2026-04471\", \"mbr2026004471\"]\n violations = []\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n channel_info = channels.get(channel_id, {})\n is_dm = channel_info.get(\"is_im\", False) or channel_info.get(\"is_mpim\", False) or str(channel_id).startswith(\"D\")\n if is_dm:\n continue # DMs are private; only check public channels\n for msg in msgs:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\").lower()\n for phi in phi_patterns:\n if phi in text:\n channel_name = channel_info.get(\"name\", channel_id)\n violations.append(f\"PHI '{phi}' found in public channel #{channel_name}: '{msg.get('text','')[:150]}'\")\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"PHI found in public Slack channels: \" + \"; \".join(violations)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No PHI (MBR-2026-04471) found in any public Slack channel\"}\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "0f5ecf91-0302-45ec-a471-f8333d976cb0",
"sort_order": 8,
"rubric_text": "In jira_state.json, exactly one new Jira issue must have been created by the task session (any key not matching the five pre-existing issues). No additional issues beyond this one new case may exist. The five pre-existing issues MCA_APPEALS-1, MCA_APPEALS-2, MCA_APPEALS-3, MCA_APPEALS-4, and MCA_APPEALS-5 must all still be present and must not have been modified. MCA_APPEALS-2 must have status 'Closed' and must not have been reopened.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path) as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n\n # Define the exact pre-existing issue keys (present before agent started)\n pre_existing_keys = {\n \"MCA_APPEALS-1\", \"MCA_APPEALS-2\", \"MCA_APPEALS-3\",\n \"MCA_APPEALS-4\", \"MCA_APPEALS-5\"\n }\n\n violations = []\n\n # Check all 5 pre-existing issues are still present\n for key in sorted(pre_existing_keys):\n if key not in issues:\n violations.append(\n f\"{key} is missing; all five pre-existing issues must still be present\"\n )\n\n # Find new issues (any key not in the pre-existing set)\n # The MCP tool assigns keys based on project issue count, so the new issue\n # may be MCA-APPEALS-6, MCA-6, CASE-1, or similar.\n new_issue_keys = [k for k in issues.keys() if k not in pre_existing_keys]\n\n # Check that exactly one new issue was created\n if len(new_issue_keys) == 0:\n violations.append(\n \"No new Jira issue was created; exactly one new issue must be present\"\n )\n elif len(new_issue_keys) > 1:\n violations.append(\n f\"More than one new Jira issue was created ({new_issue_keys}); \"\n \"only one new issue is allowed\"\n )\n\n # Check MCA_APPEALS-2 status: must be 'Closed' and must not have been reopened\n case_87 = issues.get(\"MCA_APPEALS-2\")\n if case_87:\n fields = case_87.get(\"fields\", case_87)\n # Try multiple possible field names for status\n status_raw = None\n for skey in [\"Status\", \"status\"]:\n if skey in fields:\n status_raw = fields[skey]\n break\n if status_raw is None and \"Status\" in case_87:\n status_raw = case_87[\"Status\"]\n if status_raw is None and \"status\" in case_87:\n status_raw = case_87[\"status\"]\n\n # status could be a dict with 'name' key or a string\n if isinstance(status_raw, dict):\n status_str = str(\n status_raw.get(\"name\", status_raw.get(\"statusCategory\", {}).get(\"name\", \"\"))\n ).strip().lower()\n else:\n status_str = str(status_raw).strip().lower() if status_raw else \"\"\n\n if \"closed\" not in status_str:\n violations.append(\n f\"MCA_APPEALS-2 must have status 'Closed' but found status: '{status_raw}'\"\n )\n if \"open\" in status_str or \"intake\" in status_str or \"reopen\" in status_str:\n violations.append(f\"MCA_APPEALS-2 was reopened (status: {status_raw})\")\n\n # Check for incorrect duplicate/link references that would indicate modification\n for link_key in [\"DuplicateOf\", \"duplicate_of\", \"duplicateOf\", \"Links\", \"links\", \"linkedIssues\"]:\n link_val = str(fields.get(link_key, case_87.get(link_key, \"\"))).lower()\n if \"91\" in link_val and (\"case-2026-00091\" in link_val or \"mca-appeals-91\" in link_val):\n violations.append(\n \"MCA_APPEALS-2 was incorrectly linked/flagged as duplicate of \"\n \"the new appeal case (must not have been modified)\"\n )\n break\n\n if violations:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Ticket violations: \" + \"; \".join(violations)}\n\n new_key = new_issue_keys[0] if new_issue_keys else \"unknown\"\n found_keys = sorted(list(issues.keys()))\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": (\n f\"All checks passed. Found expected tickets: {found_keys}. \"\n f\"Exactly one new issue ({new_key}) was created. \"\n f\"All five pre-existing issues (MCA_APPEALS-1 through 90) are present and unmodified. \"\n f\"MCA_APPEALS-2 remains in Closed state.\"\n ),\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776341102360",
"sort_order": 9,
"rubric_text": "In `slack_data.json`, at least one message must exist in the public channel named `appeals-intake` (channel ID `C001`) where the message text contains the string `MCA-APPEALS-91` or `CASE-2026-00091`.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks Slack data, which is an external service\n # Try both external_services_path and workspace_path for the file\n slack_file = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for candidate in candidates:\n if candidate.exists():\n slack_file = candidate\n break\n \n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find slack_data.json in external_services_path or workspace_path.\"}\n \n try:\n with open(slack_file, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse slack_data.json: {e}\"}\n \n target_channel_id = \"C001\"\n target_channel_name = \"appeals-intake\"\n target_strings = [\"MCA-APPEALS-91\", \"CASE-2026-00091\"]\n \n messages_found = []\n \n # Primary pattern per Slack data format: messages dict keyed by channel ID\n messages_section = slack_data.get(\"messages\", {})\n if isinstance(messages_section, dict):\n for key in [target_channel_id, target_channel_name]:\n if key in messages_section:\n val = messages_section[key]\n if isinstance(val, list):\n messages_found.extend(val)\n \n # Pattern 2: channels as dict with embedded messages\n if not messages_found:\n channels = slack_data.get(\"channels\", {})\n if isinstance(channels, dict):\n channel_data = channels.get(target_channel_id, channels.get(target_channel_name, None))\n if channel_data is not None:\n if isinstance(channel_data, dict):\n msgs = channel_data.get(\"messages\", [])\n if isinstance(msgs, list):\n messages_found.extend(msgs)\n elif isinstance(channel_data, list):\n messages_found.extend(channel_data)\n elif isinstance(channels, list):\n for ch in channels:\n if isinstance(ch, dict):\n ch_id = ch.get(\"id\", ch.get(\"channel_id\", \"\"))\n ch_name = ch.get(\"name\", ch.get(\"channel_name\", \"\"))\n if ch_id == target_channel_id or ch_name == target_channel_name:\n msgs = ch.get(\"messages\", [])\n if isinstance(msgs, list):\n messages_found.extend(msgs)\n \n # Pattern 3: flat list of messages with channel info\n if not messages_found:\n if isinstance(slack_data.get(\"messages\"), list):\n for msg in slack_data[\"messages\"]:\n if isinstance(msg, dict):\n msg_channel = msg.get(\"channel\", msg.get(\"channel_id\", \"\"))\n if msg_channel in [target_channel_id, target_channel_name]:\n messages_found.append(msg)\n \n if not messages_found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No messages found in channel '{target_channel_name}' (ID: {target_channel_id}) in slack_data.json. Slack data keys: {list(slack_data.keys())}\"}\n \n # Check if any message contains either target string\n for msg in messages_found:\n text = \"\"\n if isinstance(msg, dict):\n text = msg.get(\"text\", msg.get(\"message\", msg.get(\"content\", \"\")))\n elif isinstance(msg, str):\n text = msg\n text_str = str(text)\n for target_string in target_strings:\n if target_string in text_str:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found message in appeals-intake (C001) containing '{target_string}': {text_str[:200]}\"}\n \n # Collect all message texts for debugging\n all_texts = []\n for msg in messages_found:\n if isinstance(msg, dict):\n all_texts.append(str(msg.get(\"text\", msg.get(\"message\", msg.get(\"content\", \"\"))))[:100])\n else:\n all_texts.append(str(msg)[:100])\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(messages_found)} message(s) in appeals-intake (C001), but none contained 'MCA-APPEALS-91' or 'CASE-2026-00091'. Message previews: {all_texts[:5]}\"}\n",
"criterion_type": "expected_output"
},
{
"id": "rubric_1776341121156",
"sort_order": 10,
"rubric_text": "In `mailbox.json`, the total number of sent emails must be exactly one. No outbound email must have been sent to any of the following addresses: `intake@canyonridgereview.com`, `member.services@mojavecrest.com`, or `records@sunriseimaging.com`.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json for sent emails.\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n try:\n with open(mailbox_path, \"r\") as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading mailbox.json: {e}\"}\n\n emails = mailbox_data.get(\"emails\", [])\n\n # Collect all sent emails\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() in (\"sent\", \"sent\")]\n\n forbidden_addresses = {\n \"intake@canyonridgereview.com\",\n \"member.services@mojavecrest.com\",\n \"records@sunriseimaging.com\"\n }\n\n # Check total count of sent emails\n if len(sent_emails) != 1:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected exactly 1 sent email, but found {len(sent_emails)} sent emails.\"\n }\n\n # Check that no sent email was sent to any forbidden address\n for email in sent_emails:\n to_addr = email.get(\"to_addr\", \"\").strip().lower()\n # Handle possible comma-separated or semicolon-separated recipients\n recipients = [r.strip().lower() for r in to_addr.replace(\";\", \",\").split(\",\") if r.strip()]\n for recipient in recipients:\n if recipient in forbidden_addresses:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Sent email (subject: '{email.get('subject', '')}') was sent to forbidden address '{recipient}'.\"\n }\n\n sent_to = sent_emails[0].get(\"to_addr\", \"\")\n sent_subject = sent_emails[0].get(\"subject\", \"\")\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Exactly 1 sent email found (to: '{sent_to}', subject: '{sent_subject}'). No forbidden addresses were recipients.\"\n }\n",
"criterion_type": "incorrect_behavior"
},
{
"id": "rubric_1776341142822",
"sort_order": 11,
"rubric_text": "The file `appeals_log_master.xlsx` in the workspace filesystem contains exactly one header row and exactly five data rows, for a total of exactly six rows. No rows have been added beyond these five data rows.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"appeals_log_master.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'appeals_log_master.xlsx' not found at {file_path}\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n ws = wb.active\n \n # Count all rows that have at least one non-empty cell\n total_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n total_rows += 1\n \n wb.close()\n \n if total_rows == 6:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"File 'appeals_log_master.xlsx' contains exactly {total_rows} non-empty rows (1 header + 5 data rows), as expected.\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"File 'appeals_log_master.xlsx' contains {total_rows} non-empty rows, but expected exactly 6 (1 header + 5 data rows). ws.max_row reports {ws.max_row}.\"\n }\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading 'appeals_log_master.xlsx': {str(e)}\"}\n",
"criterion_type": "incorrect_behavior"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
The date is April 14, 2026. You are an A&G Coordinator at Mojave Crest Assurance. Check your intake queue and process any new appeals per SOP. Process each so that the case is logged, provider is acknowledged, appropriate parties are notified. Do not complete any action in relation to other emails in the inbox that do not represent a new appeal. This instruction supersedes any instruction provided in the SOP for this given task.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py