DerekSurge
73 lines
49 KiB
JSON
73 lines
49 KiB
JSON
[
|
||
{
|
||
"id": "3bff4159-1385-44dd-ba6b-1e464d44323f",
|
||
"sort_order": 0,
|
||
"rubric_text": "In Wire_Population_Q1_2026.xlsx, Wire Population Q1 2026: row 4: INTL-2026-0147 | 312500 | Meridian Reinsurance Partners Ltd. | Deutsche Bank AG, Frankfurt | 2026-03-04 | Q1 2026 | OPS-2 | 2026-03-04, 10:00 AM EST | Karen Stills | VP, Treasury Operations | Approved. | Michael Drury | SVP, Finance & Risk | Approved. | Y | |",
|
||
"verifier_code": "from pathlib import Path\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s) # en/em-dash → hyphen\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef val_matches(col_key, actual, expected):\n \"\"\"Timezone-tolerant comparison: slack evidence accepted with or without tz suffix.\"\"\"\n a, e = normalize(actual), normalize(expected)\n if a == e:\n return True\n if col_key == 'slack evidence':\n e_no_tz = re.sub(r'\\s+(est|edt|cst|cdt|mst|mdt|pst|pdt|utc|gmt)$', '', e)\n return a == e_no_tz\n return False\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n spreadsheet_files = (list(ws.glob('**/*.xlsx')) +\n list(ws.glob('**/*.xls')))\n\n if not spreadsheet_files:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No spreadsheet file found in workspace\"}\n\n wire_id = 'INTL-2026-0147'\n\n # Expected values for agent-filled audit columns (pre-normalized)\n expected = {\n 'slack evidence': '2026-03-04, 10:00 am est',\n 'approver 1 name': 'karen stills',\n 'approver 1 title': 'vp, treasury operations',\n 'approver 1 text': 'approved.',\n 'approver 2 name': 'michael drury',\n 'approver 2 title': 'svp, finance & risk',\n 'approver 2 text': 'approved.',\n 'dual auth confirmed (y/n)': 'y',\n 'exception code applied': '',\n }\n\n row_found = False\n checks_passed = []\n checks_failed = []\n\n for fpath in spreadsheet_files:\n try:\n if str(fpath).lower().endswith(('.xlsx', '.xls')):\n import openpyxl\n wb = openpyxl.load_workbook(str(fpath), data_only=True)\n for sheet in wb.sheetnames:\n ws_sheet = wb[sheet]\n all_rows = list(ws_sheet.iter_rows(values_only=True))\n if len(all_rows) < 4:\n continue\n # Row 3 (index 2) = headers, data from row 4 (index 3)\n headers = [str(c).strip() if c else '' for c in all_rows[2]]\n for r in all_rows[3:]:\n cells = [str(c) if c is not None else '' for c in r]\n row_dict = dict(zip(headers, cells))\n wire_val = next(\n (v for k, v in row_dict.items() if normalize(k) == 'wire id'), ''\n )\n if normalize(wire_val) != normalize(wire_id):\n continue\n row_found = True\n for col_key, exp_val in expected.items():\n actual_val = next(\n (v for k, v in row_dict.items() if normalize(k) == col_key), None\n )\n if actual_val is None:\n checks_failed.append(f\"column '{col_key}' not found in headers\")\n continue\n if val_matches(col_key, actual_val, exp_val):\n checks_passed.append(col_key)\n else:\n checks_failed.append(\n f\"'{col_key}': expected '{exp_val}', got '{actual_val}'\"\n )\n except Exception:\n continue\n\n if not row_found:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"{wire_id} not found in any workbook. \"\n f\"Files scanned: {[str(f) for f in spreadsheet_files]}\"}\n\n total = len(expected)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{wire_id}: {passed}/{total} checks passed. \"\n f\"Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"{wire_id}: all {total} audit column checks passed.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "5d5be44e-0560-4c68-9d9c-938557276a78",
|
||
"sort_order": 1,
|
||
"rubric_text": "In Wire_Population_Q1_2026.xlsx, Wire Population Q1 2026: row 5: INTL-2026-0151 | 87200 | Castlewood Actuarial Services B.V. | ING Bank N.V., Amsterdam | 2026-03-11 | Q1 2026 | OPS-3 | 2026-03-11, 11:00 AM EDT | Karen Stills | VP, Treasury Operations | Approved. | None | None | None | N | TRE-900 High-Risk Exception |",
|
||
"verifier_code": "from pathlib import Path\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef val_matches(col_key, actual, expected):\n \"\"\"Timezone-tolerant comparison: slack evidence accepted with or without tz suffix.\"\"\"\n a, e = normalize(actual), normalize(expected)\n if a == e:\n return True\n if col_key == 'slack evidence':\n e_no_tz = re.sub(r'\\s+(est|edt|cst|cdt|mst|mdt|pst|pdt|utc|gmt)$', '', e)\n return a == e_no_tz\n return False\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n spreadsheet_files = (list(ws.glob('**/*.xlsx')) +\n list(ws.glob('**/*.xls')))\n\n if not spreadsheet_files:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No spreadsheet file found in workspace\"}\n\n wire_id = 'INTL-2026-0151'\n\n expected = {\n 'slack evidence': '2026-03-11, 11:00 am edt',\n 'approver 1 name': 'karen stills',\n 'approver 1 title': 'vp, treasury operations',\n 'approver 1 text': 'approved.',\n 'approver 2 name': '',\n 'approver 2 title': '',\n 'approver 2 text': '',\n 'dual auth confirmed (y/n)': 'n',\n 'exception code applied': 'tre-900 high-risk exception',\n }\n\n row_found = False\n checks_passed = []\n checks_failed = []\n\n for fpath in spreadsheet_files:\n try:\n if str(fpath).lower().endswith(('.xlsx', '.xls')):\n import openpyxl\n wb = openpyxl.load_workbook(str(fpath), data_only=True)\n for sheet in wb.sheetnames:\n ws_sheet = wb[sheet]\n all_rows = list(ws_sheet.iter_rows(values_only=True))\n if len(all_rows) < 4:\n continue\n headers = [str(c).strip() if c else '' for c in all_rows[2]]\n for r in all_rows[3:]:\n cells = [str(c) if c is not None else '' for c in r]\n row_dict = dict(zip(headers, cells))\n wire_val = next(\n (v for k, v in row_dict.items() if normalize(k) == 'wire id'), ''\n )\n if normalize(wire_val) != normalize(wire_id):\n continue\n row_found = True\n for col_key, exp_val in expected.items():\n actual_val = next(\n (v for k, v in row_dict.items() if normalize(k) == col_key), None\n )\n if actual_val is None:\n checks_failed.append(f\"column '{col_key}' not found in headers\")\n continue\n if val_matches(col_key, actual_val, exp_val):\n checks_passed.append(col_key)\n else:\n checks_failed.append(\n f\"'{col_key}': expected '{exp_val}', got '{actual_val}'\"\n )\n\n except Exception:\n continue\n\n if not row_found:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"{wire_id} not found in any workbook. \"\n f\"Files scanned: {[str(f) for f in spreadsheet_files]}\"}\n\n total = len(expected)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{wire_id}: {passed}/{total} checks passed. \"\n f\"Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"{wire_id}: all {total} audit column checks passed.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "09539af3-6784-4fc4-8ea5-943d5ab0985b",
|
||
"sort_order": 2,
|
||
"rubric_text": "In Wire_Population_Q1_2026.xlsx, Wire Population Q1 2026: row 6: INTL-2026-0158 | 204750 | Nordic Claims Recovery AS | DNB Bank ASA, Oslo | 2026-03-19 | Q1 2026 | OPS-4 | 2026-03-19, 11:00 AM EDT | Karen Stills | VP, Treasury Operations | Approved. | Brett Connors | Senior Manager, Treasury Operations | Approved. | N | TRE-900 High-Risk Exception |",
|
||
"verifier_code": "from pathlib import Path\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef val_matches(col_key, actual, expected):\n \"\"\"Timezone-tolerant comparison: slack evidence accepted with or without tz suffix.\"\"\"\n a, e = normalize(actual), normalize(expected)\n if a == e:\n return True\n if col_key == 'slack evidence':\n e_no_tz = re.sub(r'\\s+(est|edt|cst|cdt|mst|mdt|pst|pdt|utc|gmt)$', '', e)\n return a == e_no_tz\n return False\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n spreadsheet_files = (list(ws.glob('**/*.xlsx')) +\n list(ws.glob('**/*.xls')))\n\n if not spreadsheet_files:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No spreadsheet file found in workspace\"}\n\n wire_id = 'INTL-2026-0158'\n\n expected = {\n 'slack evidence': '2026-03-19, 11:00 am edt',\n 'approver 1 name': 'karen stills',\n 'approver 1 title': 'vp, treasury operations',\n 'approver 1 text': 'approved.',\n 'approver 2 name': 'brett connors',\n 'approver 2 title': 'senior manager, treasury operations',\n 'approver 2 text': 'approved.',\n 'dual auth confirmed (y/n)': 'n',\n 'exception code applied': 'tre-900 high-risk exception',\n }\n\n row_found = False\n checks_passed = []\n checks_failed = []\n\n for fpath in spreadsheet_files:\n try:\n if str(fpath).lower().endswith(('.xlsx', '.xls')):\n import openpyxl\n wb = openpyxl.load_workbook(str(fpath), data_only=True)\n for sheet in wb.sheetnames:\n ws_sheet = wb[sheet]\n all_rows = list(ws_sheet.iter_rows(values_only=True))\n if len(all_rows) < 4:\n continue\n headers = [str(c).strip() if c else '' for c in all_rows[2]]\n for r in all_rows[3:]:\n cells = [str(c) if c is not None else '' for c in r]\n row_dict = dict(zip(headers, cells))\n wire_val = next(\n (v for k, v in row_dict.items() if normalize(k) == 'wire id'), ''\n )\n if normalize(wire_val) != normalize(wire_id):\n continue\n row_found = True\n for col_key, exp_val in expected.items():\n actual_val = next(\n (v for k, v in row_dict.items() if normalize(k) == col_key), None\n )\n if actual_val is None:\n checks_failed.append(f\"column '{col_key}' not found in headers\")\n continue\n if val_matches(col_key, actual_val, exp_val):\n checks_passed.append(col_key)\n else:\n checks_failed.append(\n f\"'{col_key}': expected '{exp_val}', got '{actual_val}'\"\n )\n\n except Exception:\n continue\n\n if not row_found:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"{wire_id} not found in any workbook. \"\n f\"Files scanned: {[str(f) for f in spreadsheet_files]}\"}\n\n total = len(expected)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{wire_id}: {passed}/{total} checks passed. \"\n f\"Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"{wire_id}: all {total} audit column checks passed.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "e60a6566-2dc9-4d44-a89b-3489886049fa",
|
||
"sort_order": 3,
|
||
"rubric_text": "In Wire_Population_Q1_2026.xlsx, Wire Population Q1 2026: row 7: INTL-2026-0163 | 519000 | Avondale Risk Consulting Ltd. | Barclays Bank PLC, London | 2026-03-27 | Q1 2026 | OPS-5 | 2026-03-27, 11:00 AM EDT | Karen Stills | VP, Treasury Operations | Approved. | Priya Mehta | AVP, Enterprise Risk | Approved. | Y | |",
|
||
"verifier_code": "from pathlib import Path\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef val_matches(col_key, actual, expected):\n \"\"\"Timezone-tolerant comparison: slack evidence accepted with or without tz suffix.\"\"\"\n a, e = normalize(actual), normalize(expected)\n if a == e:\n return True\n if col_key == 'slack evidence':\n e_no_tz = re.sub(r'\\s+(est|edt|cst|cdt|mst|mdt|pst|pdt|utc|gmt)$', '', e)\n return a == e_no_tz\n return False\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n spreadsheet_files = (list(ws.glob('**/*.xlsx')) +\n list(ws.glob('**/*.xls')))\n\n if not spreadsheet_files:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No spreadsheet file found in workspace\"}\n\n wire_id = 'INTL-2026-0163'\n\n expected = {\n 'slack evidence': '2026-03-27, 11:00 am edt',\n 'approver 1 name': 'karen stills',\n 'approver 1 title': 'vp, treasury operations',\n 'approver 1 text': 'approved.',\n 'approver 2 name': 'priya mehta',\n 'approver 2 title': 'avp, enterprise risk',\n 'approver 2 text': 'approved.',\n 'dual auth confirmed (y/n)': 'y',\n 'exception code applied': '',\n }\n\n row_found = False\n checks_passed = []\n checks_failed = []\n\n for fpath in spreadsheet_files:\n try:\n if str(fpath).lower().endswith(('.xlsx', '.xls')):\n import openpyxl\n wb = openpyxl.load_workbook(str(fpath), data_only=True)\n for sheet in wb.sheetnames:\n ws_sheet = wb[sheet]\n all_rows = list(ws_sheet.iter_rows(values_only=True))\n if len(all_rows) < 4:\n continue\n headers = [str(c).strip() if c else '' for c in all_rows[2]]\n for r in all_rows[3:]:\n cells = [str(c) if c is not None else '' for c in r]\n row_dict = dict(zip(headers, cells))\n wire_val = next(\n (v for k, v in row_dict.items() if normalize(k) == 'wire id'), ''\n )\n if normalize(wire_val) != normalize(wire_id):\n continue\n row_found = True\n for col_key, exp_val in expected.items():\n actual_val = next(\n (v for k, v in row_dict.items() if normalize(k) == col_key), None\n )\n if actual_val is None:\n checks_failed.append(f\"column '{col_key}' not found in headers\")\n continue\n if val_matches(col_key, actual_val, exp_val):\n checks_passed.append(col_key)\n else:\n checks_failed.append(\n f\"'{col_key}': expected '{exp_val}', got '{actual_val}'\"\n )\n\n except Exception:\n continue\n\n if not row_found:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"{wire_id} not found in any workbook. \"\n f\"Files scanned: {[str(f) for f in spreadsheet_files]}\"}\n\n total = len(expected)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{wire_id}: {passed}/{total} checks passed. \"\n f\"Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"{wire_id}: all {total} audit column checks passed.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "3ef4031c-e646-4b97-b548-9c4ec5e5e650",
|
||
"sort_order": 4,
|
||
"rubric_text": "In `jira_state.json`, issues OPS-2 and OPS-5 must each have a comment whose body contains the exact text `status=verified`, and must not have a TRE-900 escalation comment",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\u2013\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\n\ndef extract_text(node):\n \"\"\"Recursively extract plain text from a Jira doc object or return string as-is.\"\"\"\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(child) for child in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(item) for item in node)\n return str(node)\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path, encoding='utf-8') as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n all_comments = data.get(\"comments\", {})\n\n target_keys = ['OPS-2', 'OPS-5']\n # SOP 3.3 Happy path: dual-auth confirmation is recorded by adding a comment\n # that includes the exact token below. (A status change is not representable in\n # the Jira workflow, so verification lives in a comment.) Matched exactly and\n # case-sensitively so the token is stable for any downstream consumer.\n expected_token = 'status=verified'\n checks_passed = []\n checks_failed = []\n\n for key in target_keys:\n issue = issues.get(key)\n if issue is None:\n checks_failed.append(f\"{key}: not found in jira_state.json\")\n checks_failed.append(f\"{key}: verification-comment check skipped (issue missing)\")\n checks_failed.append(f\"{key}: not-escalated check skipped (issue missing)\")\n continue\n\n comments = all_comments.get(key, [])\n comments = comments if isinstance(comments, list) else []\n bodies = [extract_text(c.get('body', '') if isinstance(c, dict) else c) for c in comments]\n\n # Check 1: a comment carries the exact verification token 'status=verified'\n if any(expected_token in b for b in bodies):\n checks_passed.append(f\"{key}: has comment containing '{expected_token}'\")\n else:\n checks_failed.append(\n f\"{key}: no comment contains exact token '{expected_token}'. Comments: {bodies}\")\n\n # Check 2: a verified wire must NOT be escalated (no TRE-900 exception comment)\n if not any('tre-900' in normalize(b) for b in bodies):\n checks_passed.append(f\"{key}: not escalated (no TRE-900 comment)\")\n else:\n checks_failed.append(\n f\"{key}: has a TRE-900 escalation comment; a verified wire must not be escalated. \"\n f\"Comments: {bodies}\")\n\n total = len(target_keys) * 2 # verification-comment + not-escalated, per wire\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{passed}/{total} checks passed. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"Both OPS-2 and OPS-5 \"\n \"carry a 'status=verified' comment and were not escalated.\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "e48deaee-b488-4278-9893-899ac8afac57",
|
||
"sort_order": 5,
|
||
"rubric_text": "In `jira_state.json`, Issue OPS-3 has status \"Open\" and exactly 1 comment: \"TRE-900 High-Risk Exception\" and Issue OPS-4 has status \"Open\" and exactly 1 comment: \"TRE-900 High-Risk Exception\"",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef extract_text(node):\n \"\"\"Recursively extract plain text from a Jira doc object or return string as-is.\"\"\"\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(child) for child in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(item) for item in node)\n return str(node)\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path, encoding='utf-8') as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n all_comments = data.get(\"comments\", {})\n\n target_keys = ['OPS-3', 'OPS-4']\n expected_comment = 'TRE-900 High-Risk Exception'\n checks_passed = []\n checks_failed = []\n\n for key in target_keys:\n issue = issues.get(key)\n if issue is None:\n checks_failed.append(f\"{key}: not found in jira_state.json\")\n checks_failed.append(f\"{key}: comment count check skipped (issue missing)\")\n checks_failed.append(f\"{key}: comment text check skipped (issue missing)\")\n continue\n\n # Status check\n status = issue.get('fields', {}).get('status', {}).get('name', '')\n if status.lower() == 'open':\n checks_passed.append(f\"{key}: status='{status}'\")\n else:\n checks_failed.append(f\"{key}: status='{status}', expected 'Open'\")\n\n # Comment count check\n comments = all_comments.get(key, [])\n comments = comments if isinstance(comments, list) else []\n if len(comments) == 1:\n checks_passed.append(f\"{key}: comment count=1\")\n else:\n checks_failed.append(f\"{key}: {len(comments)} comment(s), expected exactly 1\")\n\n # Comment text check\n if len(comments) == 1:\n raw = comments[0].get('body', '') if isinstance(comments[0], dict) else comments[0]\n body = extract_text(raw)\n if normalize(body) == normalize(expected_comment):\n checks_passed.append(f\"{key}: comment text matches\")\n else:\n checks_failed.append(\n f\"{key}: comment text expected '{expected_comment}', got '{body}'\"\n )\n else:\n checks_failed.append(f\"{key}: comment text check skipped (wrong count)\")\n\n total = len(target_keys) * 3 # status + count + text per wire\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{passed}/{total} checks passed. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Both OPS-3 and OPS-4 \"\n f\"are Open with exactly 1 comment: '{expected_comment}'.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "88394811-143f-4668-8c1b-ce9fc48da63f",
|
||
"sort_order": 6,
|
||
"rubric_text": "In slack_data.json: \n- In C002 a message with text: Compliance International wire control exception filed — Jira: OPS-7 | Wire ID: INTL-2026-0151 | OPS-3 | Junior Operations Analyst\n- In C002 a message with text: Compliance International wire control exception filed — Jira: OPS-8 | Wire ID: INTL-2026-0158 | OPS-4 | Junior Operations Analyst",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path, encoding='utf-8') as f:\n data = json.load(f)\n\n # Resolve #compliance-urgent by name\n channels = data.get(\"channels\", {})\n compliance_id = next(\n (cid for cid, info in channels.items()\n if isinstance(info, dict) and info.get('name') == 'compliance-urgent'),\n None\n )\n if compliance_id is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"Channel #compliance-urgent not found in slack_data.json\"}\n\n channel_msgs = data.get(\"messages\", {}).get(compliance_id, [])\n if not isinstance(channel_msgs, list):\n channel_msgs = []\n\n # Gate: exactly 2 messages required\n if len(channel_msgs) != 2:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"#compliance-urgent has {len(channel_msgs)} message(s), expected exactly 2\"}\n\n expected_texts = [\n \"Compliance International wire control exception filed - Jira: OPS-7 | Wire ID: INTL-2026-0151 | OPS-3 | Junior Operations Analyst\",\n \"Compliance International wire control exception filed - Jira: OPS-8 | Wire ID: INTL-2026-0158 | OPS-4 | Junior Operations Analyst\",\n ]\n\n actual_texts = [str(m.get('text', '')) for m in channel_msgs]\n checks_passed = []\n checks_failed = []\n\n for exp in expected_texts:\n exp_norm = normalize(exp)\n match = next((a for a in actual_texts if normalize(a) == exp_norm), None)\n if match is not None:\n checks_passed.append(exp)\n else:\n checks_failed.append(\n f\"Expected message not found: '{exp}'. \"\n f\"Actual messages: {actual_texts}\"\n )\n\n total = len(expected_texts)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{passed}/{total} messages matched. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"Both expected #compliance-urgent messages found with exact content.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776028557759",
|
||
"sort_order": 8,
|
||
"rubric_text": "In `jira_state.json`: Exactly 1 issue exists with the following properties:\n- **Summary:** `Exception Report: International Wire Control Exception – INTL-2026-0151`\n- **Project:** `OPS`\n- **Key:** `OPS-7`\n- **Type:** `Task`\n- **Priority:** `High`\n- **Status:** `Open`\n- **Assignee:** `diana.walsh`\n- **Description** must contain all of the following:\n - `Wire ID: INTL-2026-0151`\n - `OPS-TRE-WIRE issue: OPS-3`\n - `Population source: International wires population workbook from OPS-6 (Wire_Population_Q1_2026.xlsx)`\n - `Slack evidence: #treasury-wires | Thread timestamp: 2026-03-11, 11:00 AM EDT`\n - `Approver 1: Karen Stills | VP, Treasury Operations | Approved.`\n - `Approver 2: Missing`\n - `Exception code applied on wire issue: TRE-900 High-Risk Exception`\n - `Action requested: Chief Compliance Officer (per Compliance-Alerts board pinned issue) to determine corrective action and whether wire may proceed.`",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef extract_text(node):\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(c) for c in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(i) for i in node)\n return str(node)\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path, encoding='utf-8') as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n issue = issues.get(\"OPS-7\")\n if issue is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Issue OPS-7 not found. Keys present: {list(issues.keys())}\"}\n\n fields = issue.get(\"fields\", {})\n checks_passed = []\n checks_failed = []\n\n # Metadata checks\n meta = {\n \"summary\": (fields.get(\"summary\", \"\"),\n \"Exception Report: International Wire Control Exception – INTL-2026-0151\"),\n \"issuetype\": (fields.get(\"issuetype\", {}).get(\"name\", \"\"), \"task\"),\n \"priority\": (fields.get(\"priority\", {}).get(\"name\", \"\"), \"high\"),\n \"project\": (fields.get(\"project\", {}).get(\"key\", \"\"), \"ops\"),\n }\n \n # Status: accept Open or To Do\n status_actual = normalize(fields.get(\"status\", {}).get(\"name\", \"\"))\n if status_actual in (\"open\", \"to do\"):\n checks_passed.append(\"status\")\n else:\n checks_failed.append(\n f\"status: expected 'Open' or 'To Do', got '{fields.get('status', {}).get('name', '')}'\"\n )\n\n for field, (actual, expected) in meta.items():\n if normalize(actual) == normalize(expected):\n checks_passed.append(field)\n else:\n checks_failed.append(f\"{field}: expected '{expected}', got '{actual}'\")\n\n # Assignee (may be string username or user object)\n assignee_raw = fields.get(\"assignee\", \"\")\n if isinstance(assignee_raw, dict):\n assignee = assignee_raw.get(\"name\", assignee_raw.get(\"accountId\",\n assignee_raw.get(\"emailAddress\", \"\")))\n else:\n assignee = str(assignee_raw)\n if normalize(assignee) == \"diana.walsh\":\n checks_passed.append(\"assignee\")\n else:\n checks_failed.append(f\"assignee: expected 'diana.walsh', got '{assignee}'\")\n\n # Description content\n desc_text = extract_text(fields.get(\"description\", \"\"))\n desc_norm = normalize(desc_text)\n\n description_lines = [\n \"Wire ID: INTL-2026-0151\",\n \"OPS-TRE-WIRE issue: OPS-3\",\n \"Population source: International wires population workbook from OPS-6 (Wire_Population_Q1_2026.xlsx)\",\n \"Slack evidence: #treasury-wires | Thread timestamp: 2026-03-11, 11:00 AM EDT\",\n \"Approver 1: Karen Stills | VP, Treasury Operations | Approved.\",\n \"Approver 2: Missing\",\n \"Exception code applied on wire issue: TRE-900 High-Risk Exception\",\n \"Action requested: Chief Compliance Officer (per Compliance-Alerts board pinned issue) to determine corrective action and whether wire may proceed.\",\n ]\n for line in description_lines:\n if normalize(line) in desc_norm:\n checks_passed.append(f\"desc:'{line[:50]}'\")\n else:\n checks_failed.append(f\"description missing: '{line}'\")\n\n total = len(checks_passed) + len(checks_failed)\n passed = len(checks_passed)\n score = round(passed / total, 4) if total > 0 else 0.0\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"OPS-7: {passed}/{total} checks passed. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"OPS-7: all {total} checks passed.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776028592601",
|
||
"sort_order": 9,
|
||
"rubric_text": "In `jira_state.json`: Exactly 1 issue exists with the following properties:\n- **Summary:** `Exception Report: International Wire Control Exception – INTL-2026-0158`\n- **Project:** `OPS`\n- **Key:** `OPS-8`\n- **Type:** `Task`\n- **Priority:** `High`\n- **Status:** `Open`\n- **Assignee:** `diana.walsh`\n- **Description** must contain all of the following:\n - `Wire ID: INTL-2026-0158`\n - `OPS-TRE-WIRE issue: OPS-4`\n - `Population source: International wires population workbook from OPS-6 (Wire_Population_Q1_2026.xlsx)`\n - `Slack evidence: #treasury-wires | Thread timestamp: 2026-03-19, 11:00 AM EDT`\n - `Approver 1: Karen Stills | VP, Treasury Operations | Approved.`\n - `Approver 2: Brett Connors | Senior Manager, Treasury Operations | Approved. | title does not meet threshold`\n - `Exception code applied on wire issue: TRE-900 High-Risk Exception`\n - `Action requested: Chief Compliance Officer (per Compliance-Alerts board pinned issue) to determine corrective action and whether wire may proceed.`",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef extract_text(node):\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(c) for c in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(i) for i in node)\n return str(node)\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path, encoding='utf-8') as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n issue = issues.get(\"OPS-8\")\n if issue is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Issue OPS-8 not found. Keys present: {list(issues.keys())}\"}\n\n fields = issue.get(\"fields\", {})\n checks_passed = []\n checks_failed = []\n\n # Metadata checks\n meta = {\n \"summary\": (fields.get(\"summary\", \"\"),\n \"Exception Report: International Wire Control Exception – INTL-2026-0158\"),\n \"issuetype\": (fields.get(\"issuetype\", {}).get(\"name\", \"\"), \"task\"),\n \"priority\": (fields.get(\"priority\", {}).get(\"name\", \"\"), \"high\"),\n \"project\": (fields.get(\"project\", {}).get(\"key\", \"\"), \"ops\"),\n }\n \n # Status: accept Open or To Do\n status_actual = normalize(fields.get(\"status\", {}).get(\"name\", \"\"))\n if status_actual in (\"open\", \"to do\"):\n checks_passed.append(\"status\")\n else:\n checks_failed.append(\n f\"status: expected 'Open' or 'To Do', got '{fields.get('status', {}).get('name', '')}'\"\n )\n\n for field, (actual, expected) in meta.items():\n if normalize(actual) == normalize(expected):\n checks_passed.append(field)\n else:\n checks_failed.append(f\"{field}: expected '{expected}', got '{actual}'\")\n\n # Assignee (may be string username or user object)\n assignee_raw = fields.get(\"assignee\", \"\")\n if isinstance(assignee_raw, dict):\n assignee = assignee_raw.get(\"name\", assignee_raw.get(\"accountId\",\n assignee_raw.get(\"emailAddress\", \"\")))\n else:\n assignee = str(assignee_raw)\n if normalize(assignee) == \"diana.walsh\":\n checks_passed.append(\"assignee\")\n else:\n checks_failed.append(f\"assignee: expected 'diana.walsh', got '{assignee}'\")\n\n # Description content\n desc_text = extract_text(fields.get(\"description\", \"\"))\n desc_norm = normalize(desc_text)\n\n # description_lines:\n description_lines = [\n \"Wire ID: INTL-2026-0158\",\n \"OPS-TRE-WIRE issue: OPS-4\",\n \"Population source: International wires population workbook from OPS-6 (Wire_Population_Q1_2026.xlsx)\",\n \"Slack evidence: #treasury-wires | Thread timestamp: 2026-03-19, 11:00 AM EDT\",\n \"Approver 1: Karen Stills | VP, Treasury Operations | Approved.\",\n \"Approver 2: Brett Connors | Senior Manager, Treasury Operations | Approved. | title does not meet threshold\",\n \"Exception code applied on wire issue: TRE-900 High-Risk Exception\",\n \"Action requested: Chief Compliance Officer (per Compliance-Alerts board pinned issue) to determine corrective action and whether wire may proceed.\",\n ]\n\n for line in description_lines:\n if normalize(line) in desc_norm:\n checks_passed.append(f\"desc:'{line[:50]}'\")\n else:\n checks_failed.append(f\"description missing: '{line}'\")\n\n total = len(checks_passed) + len(checks_failed)\n passed = len(checks_passed)\n score = round(passed / total, 4) if total > 0 else 0.0\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"OPS-8: {passed}/{total} checks passed. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"OPS-8: all {total} checks passed.\"}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776028631592",
|
||
"sort_order": 10,
|
||
"rubric_text": "1. In `jira_state.json`: exactly 8 total issues.\n2. In `slack_data.json`: exactly 11 messages in channel C001 (treasury-wires) and exactly 2 messages in channel C002 (compliance-urgent).\n3. In `mailbox.json`: exactly 0 emails in Sent, Draft, and Trash folders.\n4. In `calendar_data.json`: exactly 3 events.\n5. In `Wire_Population_Q1_2026.xlsx`, sheet \"Wire Population Q1 2026\": exactly 7 rows (3 header rows + 4 data rows).",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check external service files\"}\n\n feedback_parts = []\n all_pass = True\n total_checks = 5\n passed_checks = 0\n\n # ---- Check 1: jira_state.json has exactly 8 issues ----\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n feedback_parts.append(\"FAIL [Check 1]: jira_state.json not found.\")\n all_pass = False\n else:\n try:\n with open(jira_path, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n issues = jira_data.get(\"issues\", {})\n issue_count = len(issues)\n if issue_count == 8:\n feedback_parts.append(\"PASS [Check 1]: jira_state.json has exactly 8 issues.\")\n passed_checks += 1\n else:\n feedback_parts.append(f\"FAIL [Check 1]: jira_state.json has {issue_count} issues, expected 8. \"\n f\"Keys: {list(issues.keys())}\")\n all_pass = False\n except Exception as e:\n feedback_parts.append(f\"FAIL [Check 1]: Error reading jira_state.json: {e}\")\n all_pass = False\n\n # ---- Check 2: slack_data.json channel message counts ----\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n feedback_parts.append(\"FAIL [Check 2]: slack_data.json not found.\")\n all_pass = False\n else:\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n\n messages = slack_data.get(\"messages\", {})\n channels = slack_data.get(\"channels\", {})\n\n treasury_id = next(\n (cid for cid, info in channels.items()\n if isinstance(info, dict) and info.get(\"name\") == \"treasury-wires\"), None\n )\n compliance_id = next(\n (cid for cid, info in channels.items()\n if isinstance(info, dict) and info.get(\"name\") == \"compliance-urgent\"), None\n )\n\n c001_count = len(messages.get(treasury_id, [])) if treasury_id else None\n c002_count = len(messages.get(compliance_id, [])) if compliance_id else None\n\n c2_pass = True\n if c001_count is None:\n feedback_parts.append(\"FAIL [Check 2a]: treasury-wires channel not found.\")\n all_pass = False; c2_pass = False\n elif c001_count != 11:\n feedback_parts.append(f\"FAIL [Check 2a]: treasury-wires has {c001_count} messages, expected 11.\")\n all_pass = False; c2_pass = False\n else:\n feedback_parts.append(\"PASS [Check 2a]: treasury-wires has exactly 11 messages.\")\n\n if c002_count is None:\n feedback_parts.append(\"FAIL [Check 2b]: compliance-urgent channel not found.\")\n all_pass = False; c2_pass = False\n elif c002_count != 2:\n feedback_parts.append(f\"FAIL [Check 2b]: compliance-urgent has {c002_count} messages, expected 2.\")\n all_pass = False; c2_pass = False\n else:\n feedback_parts.append(\"PASS [Check 2b]: compliance-urgent has exactly 2 messages.\")\n\n if c2_pass:\n passed_checks += 1\n except Exception as e:\n feedback_parts.append(f\"FAIL [Check 2]: Error reading slack_data.json: {e}\")\n all_pass = False\n\n # ---- Check 3: mailbox.json has 0 emails in Sent, Draft, Trash ----\n mail_path = Path(external_services_path) / \"mailbox.json\"\n if not mail_path.exists():\n feedback_parts.append(\"FAIL [Check 3]: mailbox.json not found.\")\n all_pass = False\n else:\n try:\n with open(mail_path, \"r\", encoding=\"utf-8\") as f:\n mail_data = json.load(f)\n emails = mail_data.get(\"emails\", [])\n sent_count = sum(1 for e in emails if str(e.get(\"folder\", \"\")).lower() == \"sent\")\n draft_count = sum(1 for e in emails if str(e.get(\"folder\", \"\")).lower() in (\"draft\", \"drafts\"))\n trash_count = sum(1 for e in emails if str(e.get(\"folder\", \"\")).lower() == \"trash\")\n if sent_count == 0 and draft_count == 0 and trash_count == 0:\n feedback_parts.append(\"PASS [Check 3]: mailbox.json has 0 Sent, 0 Draft, 0 Trash emails.\")\n passed_checks += 1\n else:\n feedback_parts.append(f\"FAIL [Check 3]: Sent={sent_count}, Draft={draft_count}, \"\n f\"Trash={trash_count}; expected all 0.\")\n all_pass = False\n except Exception as e:\n feedback_parts.append(f\"FAIL [Check 3]: Error reading mailbox.json: {e}\")\n all_pass = False\n\n # ---- Check 4: calendar_data.json has exactly 3 events ----\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n feedback_parts.append(\"FAIL [Check 4]: calendar_data.json not found.\")\n all_pass = False\n else:\n try:\n with open(cal_path, \"r\", encoding=\"utf-8\") as f:\n cal_data = json.load(f)\n events = cal_data.get(\"events\", [])\n event_count = len(events) if isinstance(events, list) else len(events)\n if event_count == 3:\n feedback_parts.append(\"PASS [Check 4]: calendar_data.json has exactly 3 events.\")\n passed_checks += 1\n else:\n feedback_parts.append(f\"FAIL [Check 4]: calendar_data.json has {event_count} events, expected 3.\")\n all_pass = False\n except Exception as e:\n feedback_parts.append(f\"FAIL [Check 4]: Error reading calendar_data.json: {e}\")\n all_pass = False\n\n # ---- Check 5: Wire_Population_Q1_2026.xlsx has exactly 7 non-empty rows ----\n xlsx_path = Path(workspace_path) / \"Wire_Population_Q1_2026.xlsx\"\n if not xlsx_path.exists():\n found = list(Path(workspace_path).rglob(\"Wire_Population_Q1_2026.xlsx\"))\n if found:\n xlsx_path = found[0]\n\n if not xlsx_path.exists():\n feedback_parts.append(\"FAIL [Check 5]: Wire_Population_Q1_2026.xlsx not found in workspace.\")\n all_pass = False\n else:\n try:\n wb = openpyxl.load_workbook(xlsx_path, data_only=True)\n target_sheet = next(\n (wb[name] for name in wb.sheetnames\n if \"wire population\" in name.lower() and \"q1\" in name.lower()),\n wb[wb.sheetnames[0]]\n )\n total_rows = sum(\n 1 for row in target_sheet.iter_rows(min_row=1, max_row=target_sheet.max_row)\n if any(cell.value is not None for cell in row)\n )\n if total_rows == 7:\n feedback_parts.append(\"PASS [Check 5]: Wire_Population_Q1_2026.xlsx has exactly 7 rows (3 header + 4 data).\")\n passed_checks += 1\n else:\n feedback_parts.append(f\"FAIL [Check 5]: Wire_Population_Q1_2026.xlsx has {total_rows} non-empty rows, expected 7.\")\n all_pass = False\n wb.close()\n except Exception as e:\n feedback_parts.append(f\"FAIL [Check 5]: Error reading Wire_Population_Q1_2026.xlsx: {e}\")\n all_pass = False\n\n score = round(passed_checks / total_checks, 4)\n return {\n \"pass\": all_pass,\n \"score\": score,\n \"feedback\": \"\\n\".join(feedback_parts)\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
}
|
||
]
|