45 lines
15 KiB
JSON
45 lines
15 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "8b2fb62f-2aca-45b9-bda8-fa26cc4f08c9",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In file `PBC_Claim_Register.xlsx`, row 2 must have Claim_Status equal to 'On Hold'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n try:\n matches = list(Path(workspace_path).glob('PBC_Claim_Register.xlsx'))\n if not matches:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Claim_Register.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(matches[0])\n ws = wb.active\n # Find Claim_Status column\n header_row = [cell.value for cell in ws[1]]\n try:\n claim_status_col = header_row.index('Claim_Status') + 1\n except ValueError:\n return {'pass': False, 'score': 0.0, 'feedback': f'Claim_Status column not found. Headers: {header_row}'}\n row2_status = ws.cell(row=2, column=claim_status_col).value\n if row2_status is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 2 Claim_Status is empty/None, expected \"On Hold\"'}\n if str(row2_status).strip() == 'On Hold':\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 2 Claim_Status is \"On Hold\" as expected'}\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'Row 2 Claim_Status is \"{row2_status}\", expected \"On Hold\"'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Claim_Register.xlsx: {str(e)}'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "e93679bd-96c4-43a4-bd26-29973129e54e",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In file `PBC_Claim_Register.xlsx`, row 3 must have Claim_Status equal to 'Written Off', Billed_Amount equal to 285, Paid_Amount equal to 285, and Adjustment_Amount equal to 0.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n try:\n matches = list(Path(workspace_path).glob('PBC_Claim_Register.xlsx'))\n if not matches:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Claim_Register.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(matches[0])\n ws = wb.active\n header_row = [cell.value for cell in ws[1]]\n def get_col(name):\n try:\n return header_row.index(name) + 1\n except ValueError:\n return None\n claim_status_col = get_col('Claim_Status')\n billed_col = get_col('Billed_Amount')\n paid_col = get_col('Paid_Amount')\n adj_col = get_col('Adjustment_Amount')\n missing = [n for n, c in [('Claim_Status', claim_status_col), ('Billed_Amount', billed_col), ('Paid_Amount', paid_col), ('Adjustment_Amount', adj_col)] if c is None]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n row3_status = ws.cell(row=3, column=claim_status_col).value\n row3_billed = ws.cell(row=3, column=billed_col).value\n row3_paid = ws.cell(row=3, column=paid_col).value\n row3_adj = ws.cell(row=3, column=adj_col).value\n errors = []\n if str(row3_status).strip() != 'Written Off':\n errors.append(f'Claim_Status is \"{row3_status}\", expected \"Written Off\"')\n try:\n if abs(float(str(row3_billed).replace(',','')) - 285) > 2.85:\n errors.append(f'Billed_Amount is \"{row3_billed}\", expected 285')\n except:\n errors.append(f'Billed_Amount \"{row3_billed}\" is not numeric')\n try:\n if abs(float(str(row3_paid).replace(',','')) - 285) > 2.85:\n errors.append(f'Paid_Amount is \"{row3_paid}\", expected 285')\n except:\n errors.append(f'Paid_Amount \"{row3_paid}\" is not numeric')\n try:\n if abs(float(str(row3_adj).replace(',','')) - 0) > 2.85:\n errors.append(f'Adjustment_Amount is \"{row3_adj}\", expected 0')\n except:\n errors.append(f'Adjustment_Amount \"{row3_adj}\" is not numeric')\n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 3 issues: ' + '; '.join(errors)}\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 3 has Claim_Status=\"Written Off\", Billed_Amount={row3_billed}, Paid_Amount={row3_paid}, Adjustment_Amount={row3_adj} as expected'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Claim_Register.xlsx: {str(e)}'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "9df589fb-fede-40cb-8361-20e290f6f684",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In file `PBC_Denial_Worklist.xlsx`, row 2 must have Recovery_Action equal to 'Route to PFR' and Recovery_Status equal to 'In Progress'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n try:\n matches = list(Path(workspace_path).glob('PBC_Denial_Worklist.xlsx'))\n if not matches:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Denial_Worklist.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(matches[0])\n ws = wb.active\n header_row = [cell.value for cell in ws[1]]\n def get_col(name):\n try:\n return header_row.index(name) + 1\n except ValueError:\n return None\n action_col = get_col('Recovery_Action')\n status_col = get_col('Recovery_Status')\n missing = [n for n, c in [('Recovery_Action', action_col), ('Recovery_Status', status_col)] if c is None]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n row2_action = ws.cell(row=2, column=action_col).value\n row2_status = ws.cell(row=2, column=status_col).value\n errors = []\n if str(row2_action).strip() != 'Route to PFR':\n errors.append(f'Recovery_Action is \"{row2_action}\", expected \"Route to PFR\"')\n if str(row2_status).strip() != 'In Progress':\n errors.append(f'Recovery_Status is \"{row2_status}\", expected \"In Progress\"')\n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 2 issues: ' + '; '.join(errors)}\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 2 has Recovery_Action=\"Route to PFR\" and Recovery_Status=\"In Progress\" as expected'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Denial_Worklist.xlsx: {str(e)}'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "98830c8d-31c2-4120-8294-2c4bd73b5000",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In file `PBC_Denial_Worklist.xlsx`, row 3 must have Recovery_Action equal to 'Write Off' and Recovery_Status equal to 'Written Off'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n try:\n matches = list(Path(workspace_path).glob('PBC_Denial_Worklist.xlsx'))\n if not matches:\n return {'pass': False, 'score': 0.0, 'feedback': 'PBC_Denial_Worklist.xlsx not found in workspace'}\n wb = openpyxl.load_workbook(matches[0])\n ws = wb.active\n header_row = [cell.value for cell in ws[1]]\n def get_col(name):\n try:\n return header_row.index(name) + 1\n except ValueError:\n return None\n action_col = get_col('Recovery_Action')\n status_col = get_col('Recovery_Status')\n missing = [n for n, c in [('Recovery_Action', action_col), ('Recovery_Status', status_col)] if c is None]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found: {header_row}'}\n row3_action = ws.cell(row=3, column=action_col).value\n row3_status = ws.cell(row=3, column=status_col).value\n errors = []\n if str(row3_action).strip() != 'Write Off':\n errors.append(f'Recovery_Action is \"{row3_action}\", expected \"Write Off\"')\n if str(row3_status).strip() != 'Written Off':\n errors.append(f'Recovery_Status is \"{row3_status}\", expected \"Written Off\"')\n if errors:\n return {'pass': False, 'score': 0.0, 'feedback': 'Row 3 issues: ' + '; '.join(errors)}\n return {'pass': True, 'score': 1.0, 'feedback': f'Row 3 has Recovery_Action=\"Write Off\" and Recovery_Status=\"Written Off\" as expected'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading PBC_Denial_Worklist.xlsx: {str(e)}'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "fd59b3bf-8c01-42a6-beae-7c1ba3b035ae",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In `slack_data.json`, the #special-authorization channel must contain exactly one message with text 'Alex Rivera: CLM-2026-0062 special action authorized (or authorised)'",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n # Check both external_services_path and workspace_path for slack_data.json\n slack_path = None\n candidates = []\n if external_services_path is not None:\n p = Path(external_services_path) / 'slack_data.json'\n if p.exists():\n candidates.append(p)\n wp = Path(workspace_path) / 'slack_data.json'\n if wp.exists():\n candidates.append(wp)\n if not candidates:\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found in workspace or external_services_path'}\n slack_path = candidates[0]\n with open(slack_path, 'r') as f:\n data = json.load(f)\n\n # Find the #special-authorization channel\n channels = data.get('channels', {})\n special_auth_channel_id = None\n for ch_id, ch_info in channels.items():\n ch_name = ch_info.get('name', '')\n if 'special-authorization' in ch_name.lower() or 'special_authorization' in ch_name.lower() or 'special-authorisation' in ch_name.lower() or 'special_authorisation' in ch_name.lower():\n special_auth_channel_id = ch_id\n break\n\n if special_auth_channel_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'#special-authorization channel not found. Channels: {[v.get(\"name\") for v in channels.values()]}'}\n\n messages = data.get('messages', {})\n channel_msgs = messages.get(special_auth_channel_id, [])\n\n # Accept both \"authorized\" and \"authorised\" spellings\n # Pattern: alex rivera: clm-2026-0062 special action authorized/authorised\n # Be loose: allow extra whitespace, punctuation at end, case insensitive\n target_pattern = re.compile(\n r'alex\\s+rivera\\s*[:\\-]\\s*clm[\\-\\s]2026[\\-\\s]0062\\s+special\\s+action\\s+authori[sz]ed',\n re.IGNORECASE\n )\n\n matching = []\n for msg in channel_msgs:\n msg_text = str(msg.get('text', '')).strip()\n if target_pattern.search(msg_text):\n matching.append(msg_text)\n\n if len(matching) == 0:\n all_texts = [m.get('text', '') for m in channel_msgs]\n return {'pass': False, 'score': 0.0, 'feedback': f'No message matching \"Alex Rivera: CLM-2026-0062 special action authorized/authorised\" found in #special-authorization. Messages found: {all_texts}'}\n\n if len(matching) > 1:\n return {'pass': False, 'score': 0.5, 'feedback': f'Expected exactly 1 matching message but found {len(matching)}: {matching}'}\n\n return {'pass': True, 'score': 1.0, 'feedback': f'Found exactly 1 message in #special-authorization matching the required text: \"{matching[0]}\"'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading slack_data.json: {str(e)}'}\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775144961819",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "There must be no sent emails in the file `mailbox.json`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Look for mailbox.json in external_services_path first, then workspace_path\n mailbox_path = None\n\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if mailbox_path is None:\n # No mailbox.json found at all — no sent emails can exist\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No mailbox.json file found. No sent emails present.\"}\n\n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except json.JSONDecodeError as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error reading mailbox.json: {e}\"}\n\n # Extract the list of emails from various possible structures\n emails = []\n if isinstance(data, dict):\n if \"emails\" in data:\n emails = data[\"emails\"]\n elif \"messages\" in data:\n emails = data[\"messages\"]\n else:\n for key, value in data.items():\n if isinstance(value, list):\n emails = value\n break\n elif isinstance(data, list):\n emails = data\n\n if not isinstance(emails, list):\n emails = []\n\n # Filter to only SENT emails\n # Be generous: check multiple possible field names and values for \"sent\"\n sent_emails = []\n for e in emails:\n if not isinstance(e, dict):\n continue\n folder = e.get(\"folder\", e.get(\"Folder\", e.get(\"type\", e.get(\"status\", \"\")))).strip().lower() if isinstance(e.get(\"folder\", e.get(\"Folder\", e.get(\"type\", e.get(\"status\", \"\")))), str) else \"\"\n if folder in (\"sent\", \"outbox\", \"sent mail\", \"sent items\"):\n sent_emails.append(e)\n\n num_sent = len(sent_emails)\n\n if num_sent == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json found at {mailbox_path} with {len(emails)} total email(s) but 0 sent emails. Pass.\"}\n else:\n # Summarize sent emails for feedback\n subjects = []\n for e in sent_emails[:5]:\n if isinstance(e, dict):\n subj = e.get(\"subject\", e.get(\"Subject\", \"(no subject)\"))\n subjects.append(str(subj))\n subjects_str = \", \".join(subjects)\n if num_sent > 5:\n subjects_str += f\", ... and {num_sent - 5} more\"\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json contains {num_sent} sent email(s). Expected 0 sent emails. Sent email subjects: [{subjects_str}]. File: {mailbox_path}\"}\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
}
|
||
|
|
]
|