DerekSurge
87 lines
57 KiB
JSON
87 lines
57 KiB
JSON
[
|
||
{
|
||
"id": "b1ba5aca-eaef-4309-bb8c-dd98a2c055f0",
|
||
"sort_order": 0,
|
||
"rubric_text": "In `Wire_Population_Q1_2026.xlsx`, the 'Dual Auth Confirmed (Y/N)' column (headers in row 3) must be 'Y' for INTL-2026-0214 and INTL-2026-0219, and 'N' for INTL-2026-0223 and INTL-2026-0228.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target = None\n for f in xlsx_files:\n if 'wire' in f.name.lower() and 'population' in f.name.lower():\n target = f\n break\n if target is None:\n for f in xlsx_files:\n if 'wire' in f.name.lower():\n target = f\n break\n if target is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Wire Population xlsx file found in workspace.'}\n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open workbook: {e}'}\n\n # The rubric says headers are in row 3. Try rows 1, 2, and 3 to find the header row.\n header_row = None\n wire_id_col = None\n dual_auth_col = None\n\n for candidate_row in [3, 1, 2, 4, 5]:\n try:\n headers = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[candidate_row]]\n except Exception:\n continue\n temp_wire_id_col = None\n temp_dual_auth_col = None\n for i, h in enumerate(headers):\n hl = h.lower()\n if ('wire' in hl and ('id' in hl or 'ref' in hl or 'num' in hl)) or 'wire id' in hl or 'reference' in hl:\n temp_wire_id_col = i\n if 'dual' in hl and 'auth' in hl:\n temp_dual_auth_col = i\n # Also try matching on just column content patterns\n if temp_wire_id_col is None:\n for i, h in enumerate(headers):\n if 'intl' in h.upper() or h.upper().startswith('WIRE'):\n temp_wire_id_col = i\n break\n if temp_dual_auth_col is not None:\n header_row = candidate_row\n wire_id_col = temp_wire_id_col\n dual_auth_col = temp_dual_auth_col\n break\n\n if dual_auth_col is None:\n # Fallback: scan all columns across all rows for the header\n all_headers_info = []\n for r in range(1, min(ws.max_row + 1, 10)):\n row_vals = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[r]]\n all_headers_info.append(f'Row {r}: {row_vals}')\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Dual Auth Confirmed column. Scanned rows: {\" | \".join(all_headers_info)}'}\n\n if wire_id_col is None:\n wire_id_col = 0 # default to first column\n\n expected = {\n 'INTL-2026-0214': 'Y',\n 'INTL-2026-0219': 'Y',\n 'INTL-2026-0223': 'N',\n 'INTL-2026-0228': 'N',\n }\n results = {}\n for row in ws.iter_rows(min_row=header_row + 1, values_only=True):\n if len(row) <= max(wire_id_col, dual_auth_col):\n continue\n wire_id = str(row[wire_id_col]).strip() if row[wire_id_col] is not None else ''\n if wire_id in expected:\n val = str(row[dual_auth_col]).strip().upper() if row[dual_auth_col] is not None else ''\n results[wire_id] = val\n\n failures = []\n for wire_id, exp_val in expected.items():\n actual_val = results.get(wire_id, 'NOT FOUND')\n if actual_val != exp_val:\n failures.append(f'{wire_id}: expected {exp_val}, got {actual_val}')\n\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'Dual Auth Confirmed mismatches: {failures}. Found results: {results}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Dual Auth Confirmed values correct for all four wire IDs (headers in row {header_row}). Found: {results}'}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "777eb2ba-7c99-4eab-ad48-0d7dbed3a938",
|
||
"sort_order": 1,
|
||
"rubric_text": "In `Wire_Population_Q1_2026.xlsx`, the 'Exception Code Applied' column (headers in row 3) must contain 'TRE-900' for INTL-2026-0223 and INTL-2026-0228, and must NOT contain 'TRE-900' for INTL-2026-0214 and INTL-2026-0219.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target = None\n for f in xlsx_files:\n if 'wire' in f.name.lower() and 'population' in f.name.lower():\n target = f\n break\n if target is None:\n for f in xlsx_files:\n if 'wire' in f.name.lower():\n target = f\n break\n if target is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Wire Population xlsx file found.'}\n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open workbook: {e}'}\n\n # The rubric says headers are in row 3. We'll try rows 1, 2, and 3 to find the right header row.\n header_row = None\n exc_col = None\n wire_id_col = None\n\n for candidate_row in [3, 1, 2, 4, 5]:\n row_cells = list(ws.iter_rows(min_row=candidate_row, max_row=candidate_row, values_only=True))\n if not row_cells:\n continue\n headers = [str(cell).strip() if cell is not None else '' for cell in row_cells[0]]\n tmp_exc = None\n tmp_wire = None\n for i, h in enumerate(headers):\n hl = h.lower()\n if 'exception' in hl and 'code' in hl:\n tmp_exc = i\n if ('wire' in hl and ('id' in hl or 'ref' in hl or 'num' in hl)) or 'transaction' in hl and 'id' in hl:\n tmp_wire = i\n if h.upper().startswith('INTL'):\n tmp_wire = i\n if tmp_exc is not None:\n exc_col = tmp_exc\n wire_id_col = tmp_wire\n header_row = candidate_row\n break\n\n if exc_col is None:\n # Last resort: scan all columns across first 5 rows\n all_headers_info = []\n for r in range(1, 6):\n row_cells = list(ws.iter_rows(min_row=r, max_row=r, values_only=True))\n if row_cells:\n all_headers_info.append((r, [str(c).strip() if c is not None else '' for c in row_cells[0]]))\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Exception Code Applied column. Scanned rows: {all_headers_info}'}\n\n if wire_id_col is None:\n wire_id_col = 0\n\n data_start_row = header_row + 1\n\n must_have = {'INTL-2026-0223', 'INTL-2026-0228'}\n must_not_have = {'INTL-2026-0214', 'INTL-2026-0219'}\n results = {}\n for row in ws.iter_rows(min_row=data_start_row, values_only=True):\n if row is None:\n continue\n wire_id_raw = row[wire_id_col] if wire_id_col < len(row) else None\n wire_id = str(wire_id_raw).strip() if wire_id_raw is not None else ''\n if wire_id in must_have or wire_id in must_not_have:\n exc_val_raw = row[exc_col] if exc_col < len(row) else None\n val = str(exc_val_raw).strip() if exc_val_raw is not None else ''\n results[wire_id] = val\n\n failures = []\n for wid in must_have:\n val = results.get(wid, 'NOT FOUND')\n if 'TRE-900' not in val.upper().replace(' ', ''):\n # also try without dash\n if 'TRE900' not in val.upper().replace(' ', '').replace('-', ''):\n failures.append(f'{wid}: expected TRE-900 in exception code, got \"{val}\"')\n for wid in must_not_have:\n val = results.get(wid, '')\n if 'TRE-900' in val.upper().replace(' ', '') or 'TRE900' in val.upper().replace(' ', '').replace('-', ''):\n failures.append(f'{wid}: TRE-900 should NOT be present, got \"{val}\"')\n\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'Exception Code Applied issues: {failures}. Header row detected: {header_row}. Results found: {results}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Exception Code Applied correct for all four wire IDs. Header row: {header_row}. Found: {results}'}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "d49db332-325f-4ecf-bfbc-c95e2872ec9d",
|
||
"sort_order": 2,
|
||
"rubric_text": "In `Wire_Population_Q1_2026.xlsx`, INTL-2026-0214 must have 'claire.overton' (with or without \".\", case insensitive) in 'Approver 1 Name' (headers in row 3) with 'Approved.' in 'Approver 1 Text', and 'rachel.kim' (with or without \".\", case insensitive) in 'Approver 2 Name' with 'Approved.' in 'Approver 2 Text'. INTL-2026-0219 must have 'peter.walsh' (with or without \".\", case insensitive) in 'Approver 1 Name' with 'Approved.' in 'Approver 1 Text', and 'nadine.cole' (with or without \".\", case insensitive) in 'Approver 2 Name' with 'Approved.' in 'Approver 2 Text'.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target = None\n for f in xlsx_files:\n if 'wire' in f.name.lower() and 'population' in f.name.lower():\n target = f\n break\n if target is None:\n for f in xlsx_files:\n if 'wire' in f.name.lower():\n target = f\n break\n if target is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Wire Population xlsx file found.'}\n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open workbook: {e}'}\n\n # The rubric says headers are in row 3. Try rows 1-5 to find headers.\n header_row = None\n headers = []\n for candidate_row in [3, 1, 2, 4, 5]:\n row_cells = list(ws[candidate_row])\n candidate_headers = [str(cell.value).strip().lower() if cell.value is not None else '' for cell in row_cells]\n if any('approver' in h for h in candidate_headers):\n headers = candidate_headers\n header_row = candidate_row\n break\n\n if header_row is None:\n row_cells = list(ws[3])\n headers = [str(cell.value).strip().lower() if cell.value is not None else '' for cell in row_cells]\n header_row = 3\n\n def find_col(keywords):\n for i, h in enumerate(headers):\n if all(k.lower() in h for k in keywords):\n return i\n return None\n\n # Try to find wire ID column\n wire_id_col = find_col(['wire', 'id'])\n if wire_id_col is None:\n wire_id_col = find_col(['wire'])\n if wire_id_col is None:\n wire_id_col = find_col(['intl'])\n if wire_id_col is None:\n wire_id_col = find_col(['id'])\n if wire_id_col is None:\n wire_id_col = 0\n\n ap1_name_col = find_col(['approver', '1', 'name'])\n ap1_text_col = find_col(['approver', '1', 'text'])\n ap2_name_col = find_col(['approver', '2', 'name'])\n ap2_text_col = find_col(['approver', '2', 'text'])\n\n # Try alternative matching with broader keywords\n if None in [ap1_name_col, ap1_text_col, ap2_name_col, ap2_text_col]:\n for i, h in enumerate(headers):\n if 'approver' in h and '1' in h and 'name' in h and ap1_name_col is None:\n ap1_name_col = i\n elif 'approver' in h and '1' in h and ('text' in h or 'comment' in h or 'response' in h or 'status' in h or 'decision' in h) and ap1_text_col is None:\n ap1_text_col = i\n elif 'approver' in h and '2' in h and 'name' in h and ap2_name_col is None:\n ap2_name_col = i\n elif 'approver' in h and '2' in h and ('text' in h or 'comment' in h or 'response' in h or 'status' in h or 'decision' in h) and ap2_text_col is None:\n ap2_text_col = i\n\n missing_cols = [name for name, col in [('Approver 1 Name', ap1_name_col), ('Approver 1 Text', ap1_text_col), ('Approver 2 Name', ap2_name_col), ('Approver 2 Text', ap2_text_col)] if col is None]\n if missing_cols:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find columns: {missing_cols}. Headers found in row {header_row}: {headers}'}\n\n def name_matches(actual, expected_dotted):\n \"\"\"Check if actual name matches expected, with or without dot, case insensitive.\n E.g., 'claire.overton' should match 'claireoverton', 'claire.overton', 'Claire.Overton', 'Claire Overton', etc.\n \"\"\"\n actual_clean = actual.lower().strip()\n # Remove dots and spaces for comparison\n actual_nodot = actual_clean.replace('.', '').replace(' ', '')\n expected_nodot = expected_dotted.lower().replace('.', '').replace(' ', '')\n # Check: exact with dot, without dot, or contained in\n if expected_dotted.lower() in actual_clean:\n return True\n if expected_nodot in actual_nodot:\n return True\n return False\n\n def text_matches(actual):\n \"\"\"Check if the text contains 'approved' (case insensitive), being lenient.\"\"\"\n return 'approved' in actual.lower().strip()\n\n expected = {\n 'INTL-2026-0214': {'ap1_name': 'claire.overton', 'ap2_name': 'rachel.kim'},\n 'INTL-2026-0219': {'ap1_name': 'peter.walsh', 'ap2_name': 'nadine.cole'},\n }\n failures = []\n found = {}\n\n data_start_row = header_row + 1\n\n for row in ws.iter_rows(min_row=data_start_row, values_only=False):\n row_vals = [cell.value for cell in row]\n wire_id_val = row_vals[wire_id_col] if wire_id_col < len(row_vals) else None\n wire_id = str(wire_id_val).strip() if wire_id_val is not None else ''\n if wire_id in expected:\n ap1n = str(row_vals[ap1_name_col]).strip() if ap1_name_col < len(row_vals) and row_vals[ap1_name_col] is not None else ''\n ap1t = str(row_vals[ap1_text_col]).strip() if ap1_text_col < len(row_vals) and row_vals[ap1_text_col] is not None else ''\n ap2n = str(row_vals[ap2_name_col]).strip() if ap2_name_col < len(row_vals) and row_vals[ap2_name_col] is not None else ''\n ap2t = str(row_vals[ap2_text_col]).strip() if ap2_text_col < len(row_vals) and row_vals[ap2_text_col] is not None else ''\n found[wire_id] = {'ap1_name': ap1n, 'ap1_text': ap1t, 'ap2_name': ap2n, 'ap2_text': ap2t}\n exp = expected[wire_id]\n if not name_matches(ap1n, exp['ap1_name']):\n failures.append(f'{wire_id}: Approver 1 Name expected to contain \"{exp[\"ap1_name\"]}\" (with or without dot), got \"{ap1n}\"')\n if not text_matches(ap1t):\n failures.append(f'{wire_id}: Approver 1 Text expected to contain \"Approved.\", got \"{ap1t}\"')\n if not name_matches(ap2n, exp['ap2_name']):\n failures.append(f'{wire_id}: Approver 2 Name expected to contain \"{exp[\"ap2_name\"]}\" (with or without dot), got \"{ap2n}\"')\n if not text_matches(ap2t):\n failures.append(f'{wire_id}: Approver 2 Text expected to contain \"Approved.\", got \"{ap2t}\"')\n\n for wid in expected:\n if wid not in found:\n failures.append(f'{wid}: row not found in workbook (searched from row {data_start_row} onward with wire ID in column {wire_id_col})')\n\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'Approver field mismatches: {failures}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Approver fields correct for INTL-2026-0214 and INTL-2026-0219. Found: {found}'}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "cea31bfb-2464-455f-a17e-91fed2c70511",
|
||
"sort_order": 3,
|
||
"rubric_text": "In `Wire_Population_Q1_2026.xlsx`, INTL-2026-0223 must have 'claire.overton' (with or without \".\", case insensitive) in 'Approver 1 Name' (headers in row 3) with 'Approved.' in 'Approver 1 Text', and 'steven.ford' (with or without \".\", case insensitive) with response 'LGTM' must appear somewhere in the row (any approver text or notes column). INTL-2026-0228 must have 'claire.overton' (with or without \".\", case insensitive) in 'Approver 1 Name' with 'Approved.' in 'Approver 1 Text', and 'Approver 2 Name' must be blank/N/A/Missing/empty.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target = None\n for f in xlsx_files:\n if 'wire' in f.name.lower() and 'population' in f.name.lower():\n target = f\n break\n if target is None:\n for f in xlsx_files:\n if 'wire' in f.name.lower():\n target = f\n break\n if target is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Wire Population xlsx file found.'}\n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open workbook: {e}'}\n\n # The rubric says headers are in row 3, so try rows 1-5 to find headers\n header_row = None\n headers = []\n for candidate_row in [3, 1, 2, 4, 5]:\n if candidate_row > ws.max_row:\n continue\n candidate_headers = [str(cell.value).strip().lower() if cell.value is not None else '' for cell in ws[candidate_row]]\n has_approver = any('approver' in h for h in candidate_headers)\n if has_approver:\n headers = candidate_headers\n header_row = candidate_row\n break\n if header_row is None:\n header_row = 3\n if header_row <= ws.max_row:\n headers = [str(cell.value).strip().lower() if cell.value is not None else '' for cell in ws[header_row]]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'Worksheet has only {ws.max_row} rows, cannot find header row.'}\n\n wire_id_col = None\n ap1_name_col = None\n ap1_text_col = None\n ap2_name_col = None\n ap2_text_col = None\n notes_cols = []\n all_approver_text_cols = []\n all_approver_name_cols = []\n\n for i, h in enumerate(headers):\n # Wire ID column detection\n if wire_id_col is None and (\n ('wire' in h and ('id' in h or 'ref' in h or 'num' in h)) or\n h.startswith('intl') or 'wire id' in h or 'reference' in h or\n 'transaction' in h or 'wire_id' in h\n ):\n wire_id_col = i\n # Approver 1 Name\n if 'approver' in h and '1' in h and 'name' in h and ap1_name_col is None:\n ap1_name_col = i\n # Approver 1 Text\n if 'approver' in h and '1' in h and ('text' in h or 'comment' in h or 'response' in h) and ap1_text_col is None:\n ap1_text_col = i\n # Approver 2 Name\n if 'approver' in h and '2' in h and 'name' in h and ap2_name_col is None:\n ap2_name_col = i\n # Approver 2 Text\n if 'approver' in h and '2' in h and ('text' in h or 'comment' in h or 'response' in h) and ap2_text_col is None:\n ap2_text_col = i\n # Track all approver text and name columns\n if 'approver' in h and ('text' in h or 'comment' in h or 'response' in h):\n all_approver_text_cols.append(i)\n if 'approver' in h and 'name' in h:\n all_approver_name_cols.append(i)\n # Notes columns\n if 'note' in h or 'comment' in h or 'remark' in h or 'flag' in h or 'finding' in h:\n notes_cols.append(i)\n\n # If wire_id_col not found, try first column\n if wire_id_col is None:\n wire_id_col = 0\n\n missing = [name for name, col in [\n ('Approver 1 Name', ap1_name_col),\n ('Approver 1 Text', ap1_text_col),\n ('Approver 2 Name', ap2_name_col)\n ] if col is None]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing columns: {missing}. Headers found in row {header_row}: {headers}'}\n\n def cell_val(row_tuple, col_idx):\n \"\"\"Safely get cell value from row tuple.\"\"\"\n if col_idx is not None and col_idx < len(row_tuple) and row_tuple[col_idx] is not None:\n return str(row_tuple[col_idx]).strip()\n return ''\n\n def name_matches(val, target_name):\n \"\"\"Check if val matches the target name, with or without dot, case insensitive.\n e.g., 'claire.overton' matches 'claireoverton', 'claire.overton', 'Claire.Overton', etc.\"\"\"\n val_clean = val.lower().replace('.', '').replace(' ', '')\n target_clean = target_name.lower().replace('.', '').replace(' ', '')\n return target_clean in val_clean\n\n failures = []\n found_0223 = False\n found_0228 = False\n\n for row in ws.iter_rows(min_row=header_row + 1, values_only=True):\n if row is None:\n continue\n wire_id_val = row[wire_id_col] if wire_id_col < len(row) else None\n wire_id = str(wire_id_val).strip() if wire_id_val is not None else ''\n\n # Build a string of the entire row for loose searching\n row_str = ' '.join(str(c) for c in row if c is not None)\n\n # ---- INTL-2026-0223 ----\n if 'INTL-2026-0223' in wire_id.upper() or wire_id.upper() == 'INTL-2026-0223':\n found_0223 = True\n ap1n = cell_val(row, ap1_name_col).lower()\n ap1t = cell_val(row, ap1_text_col)\n\n # Check Approver 1 Name contains claire.overton (with or without dot)\n if not name_matches(ap1n, 'claire.overton'):\n failures.append(f'INTL-2026-0223: Approver 1 Name should contain claire.overton (case insensitive, with/without dot), got: \"{ap1n}\"')\n\n # Check Approver 1 Text contains 'Approved.' (loose: check for 'approved')\n if 'approved' not in ap1t.lower():\n failures.append(f'INTL-2026-0223: Approver 1 Text should contain \"Approved.\", got: \"{ap1t}\"')\n\n # Check steven.ford appears somewhere in row (any approver name/text or notes column)\n row_str_lower = row_str.lower()\n steven_found = name_matches(row_str_lower, 'steven.ford')\n if not steven_found:\n # Also check individual relevant columns\n for ci in all_approver_name_cols + all_approver_text_cols + notes_cols:\n cv = cell_val(row, ci).lower()\n if name_matches(cv, 'steven.ford'):\n steven_found = True\n break\n if not steven_found:\n failures.append(f'INTL-2026-0223: steven.ford not found anywhere in the row')\n\n # Check LGTM appears somewhere in the row\n lgtm_found = 'lgtm' in row_str_lower\n if not lgtm_found:\n for ci in all_approver_text_cols + notes_cols:\n cv = cell_val(row, ci).lower()\n if 'lgtm' in cv:\n lgtm_found = True\n break\n if not lgtm_found:\n failures.append(f'INTL-2026-0223: \"LGTM\" not found anywhere in the row')\n\n # ---- INTL-2026-0228 ----\n if 'INTL-2026-0228' in wire_id.upper() or wire_id.upper() == 'INTL-2026-0228':\n found_0228 = True\n ap1n = cell_val(row, ap1_name_col).lower()\n ap1t = cell_val(row, ap1_text_col)\n ap2n = cell_val(row, ap2_name_col)\n\n # Check Approver 1 Name contains claire.overton\n if not name_matches(ap1n, 'claire.overton'):\n failures.append(f'INTL-2026-0228: Approver 1 Name should contain claire.overton (case insensitive, with/without dot), got: \"{ap1n}\"')\n\n # Check Approver 1 Text contains 'Approved.'\n if 'approved' not in ap1t.lower():\n failures.append(f'INTL-2026-0228: Approver 1 Text should contain \"Approved.\", got: \"{ap1t}\"')\n\n # Approver 2 Name must be blank/N/A/Missing/empty\n ap2n_lower = ap2n.lower().strip()\n is_blank_or_missing = (\n ap2n_lower == '' or\n ap2n_lower in ['n/a', 'na', 'missing', 'none', 'null', '-', 'empty', '—', 'n/ a'] or\n 'missing' in ap2n_lower or\n ap2n_lower == 'none' or\n ap2n_lower == 'n/a'\n )\n if not is_blank_or_missing:\n failures.append(f'INTL-2026-0228: Approver 2 Name should be blank/N/A/Missing, got: \"{ap2n}\"')\n\n if not found_0223:\n failures.append('INTL-2026-0223 row not found in workbook')\n if not found_0228:\n failures.append('INTL-2026-0228 row not found in workbook')\n\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'Issues found: {failures}'}\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (\n f'INTL-2026-0223 and INTL-2026-0228 approver fields verified correctly. '\n f'Headers found in row {header_row}. File: {target.name}'\n )\n }\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "90e5d5f1-b719-4d7e-8063-86eb1c0deed3",
|
||
"sort_order": 4,
|
||
"rubric_text": "In `Wire_Population_Q1_2026.xlsx`, the 'Slack Query Used' column (headers in row 3) for INTL-2026-0219 must NOT contain only 'has:link INTL-2026-0219'; it must contain 'in:#treasury-wires' or indicate query (2) or (3) was used.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_files = list(Path(workspace_path).glob('**/*.xlsx'))\n target = None\n for f in xlsx_files:\n if 'wire' in f.name.lower() and 'population' in f.name.lower():\n target = f\n break\n if target is None:\n for f in xlsx_files:\n if 'wire' in f.name.lower():\n target = f\n break\n if target is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No Wire Population xlsx file found.'}\n try:\n wb = openpyxl.load_workbook(target, data_only=True)\n ws = wb.active\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to open workbook: {e}'}\n\n # The rubric says headers are in row 3, so try rows 1, 2, and 3 for headers\n header_row = None\n headers = []\n for candidate_row in [3, 1, 2, 4]:\n candidate_headers = [str(cell.value).strip().lower() if cell.value is not None else '' for cell in ws[candidate_row]]\n has_slack = any('slack' in h and 'query' in h for h in candidate_headers)\n if has_slack:\n headers = candidate_headers\n header_row = candidate_row\n break\n if header_row is None:\n # Fallback: try all rows up to 10\n for candidate_row in range(1, 11):\n candidate_headers = [str(cell.value).strip().lower() if cell.value is not None else '' for cell in ws[candidate_row]]\n has_slack = any('slack' in h and 'query' in h for h in candidate_headers)\n if has_slack:\n headers = candidate_headers\n header_row = candidate_row\n break\n if header_row is None:\n all_headers_tried = []\n for r in range(1, 6):\n row_vals = [str(cell.value).strip().lower() if cell.value is not None else '' for cell in ws[r]]\n all_headers_tried.append(f'Row {r}: {row_vals}')\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Slack Query Used column in any header row. Tried rows: {chr(10).join(all_headers_tried)}'}\n\n wire_id_col = None\n slack_query_col = None\n for i, h in enumerate(headers):\n if ('wire' in h and ('id' in h or 'ref' in h or 'num' in h)) or h.startswith('intl') or 'transaction' in h or h == 'id':\n wire_id_col = i\n if 'slack' in h and 'query' in h:\n slack_query_col = i\n\n if slack_query_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Slack Query Used column. Headers (row {header_row}): {headers}'}\n\n # If we couldn't identify wire id column, try first column or scan for one containing 'INTL'\n if wire_id_col is None:\n # Try to find a column that has INTL-2026 values\n for row in ws.iter_rows(min_row=header_row + 1, max_row=header_row + 5, values_only=True):\n for ci, cell_val in enumerate(row):\n if cell_val and 'INTL-2026' in str(cell_val):\n wire_id_col = ci\n break\n if wire_id_col is not None:\n break\n if wire_id_col is None:\n wire_id_col = 0 # fallback to first column\n\n found_row = False\n for row in ws.iter_rows(min_row=header_row + 1, values_only=True):\n wire_id = str(row[wire_id_col]).strip() if row[wire_id_col] is not None else ''\n if wire_id == 'INTL-2026-0219':\n found_row = True\n val = str(row[slack_query_col]).strip() if slack_query_col < len(row) and row[slack_query_col] is not None else ''\n val_lower = val.lower()\n\n # Check for positive indicators: in:#treasury-wires, query (2), query (3)\n if 'in:#treasury-wires' in val_lower or '#treasury-wires' in val_lower:\n return {'pass': True, 'score': 1.0, 'feedback': f'INTL-2026-0219 Slack Query Used correctly contains #treasury-wires: \"{val}\"'}\n if 'query (2)' in val_lower or 'query (3)' in val_lower or 'query 2' in val_lower or 'query 3' in val_lower:\n return {'pass': True, 'score': 1.0, 'feedback': f'INTL-2026-0219 Slack Query Used correctly references query 2 or 3: \"{val}\"'}\n # Also accept if it contains treasury-wires in any form\n if 'treasury-wires' in val_lower or 'treasury_wires' in val_lower:\n return {'pass': True, 'score': 1.0, 'feedback': f'INTL-2026-0219 Slack Query Used contains treasury-wires reference: \"{val}\"'}\n\n # Check it's not ONLY the first query\n normalized = val.lower().strip()\n only_first = normalized in ['has:link intl-2026-0219', 'has:link intl-2026-0219']\n if only_first or (normalized and 'in:' not in normalized and '#' not in normalized and 'query' not in normalized and 'treasury' not in normalized):\n return {'pass': False, 'score': 0.0, 'feedback': f'INTL-2026-0219 Slack Query Used does not reflect query (2) or (3) with in:#treasury-wires. Got: \"{val}\"'}\n\n return {'pass': False, 'score': 0.0, 'feedback': f'INTL-2026-0219 Slack Query Used does not clearly indicate query (2) or (3) with in:#treasury-wires. Got: \"{val}\"'}\n\n if not found_row:\n return {'pass': False, 'score': 0.0, 'feedback': f'INTL-2026-0219 row not found in workbook. Wire ID column index: {wire_id_col}, header row: {header_row}'}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "d90a6cf3-184d-4712-b2d2-b8af81edfa59",
|
||
"sort_order": 5,
|
||
"rubric_text": "In `jira_state.json`, OPS-1 and OPS-2 must each have a comment whose body contains the exact text `status=verified`. OPS-3 and OPS-4 must NOT have any comment containing `status=verified` (they are exception wires requiring CCO review, not confirmed dual-auth wires).",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef extract_text(node):\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(c) for c in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(i) for i in node)\n return str(node)\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided; cannot check jira_state.json.'}\n jira_path = Path(external_services_path) / 'jira_state.json'\n if not jira_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path.'}\n try:\n with open(jira_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse jira_state.json: {e}'}\n\n issues = data.get('issues', data) if isinstance(data, dict) else {}\n all_comments = data.get('comments', {}) if isinstance(data, dict) else {}\n token = 'status=verified' # exact verification marker from SOP 3.3 Happy path\n failures = []\n\n def find_issue(key):\n issue = issues.get(key)\n if issue is None:\n for k, v in issues.items():\n if isinstance(v, dict) and v.get('key', '').upper() == key.upper():\n return v\n return issue\n\n def has_verify_comment(key):\n comments = all_comments.get(key, [])\n comments = comments if isinstance(comments, list) else []\n for c in comments:\n body = extract_text(c.get('body', '') if isinstance(c, dict) else c)\n if token in body:\n return True\n return False\n\n # 0214 and 0219 (dual-auth confirmed) MUST carry a 'status=verified' comment\n for key in ['OPS-1', 'OPS-2']:\n if find_issue(key) is None:\n failures.append(f'{key}: issue not found in jira_state.json')\n continue\n if not has_verify_comment(key):\n failures.append(f\"{key}: no comment contains exact token '{token}', expected one\")\n\n # 0223 and 0228 (exception wires) MUST NOT carry a 'status=verified' comment\n for key in ['OPS-3', 'OPS-4']:\n if find_issue(key) is None:\n failures.append(f'{key}: issue not found in jira_state.json')\n continue\n if has_verify_comment(key):\n failures.append(f\"{key}: has a '{token}' comment but this is an exception wire — must not be marked verified\")\n\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'Jira verification-comment issues: {failures}'}\n return {'pass': True, 'score': 1.0, 'feedback': \"OPS-1 and -0219 carry a 'status=verified' comment; -0223 and -0228 do not.\"}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "f27e4f15-5e0a-4c91-8aee-c0ea4181ef9d",
|
||
"sort_order": 6,
|
||
"rubric_text": "In `jira_state.json`, OPS-3 and OPS-4 must each have 'TRE-900' applied as a label, in a comment, or in a custom field (NOT only in the description field).",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided.'}\n jira_path = Path(external_services_path) / 'jira_state.json'\n if not jira_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found.'}\n try:\n with open(jira_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse jira_state.json: {e}'}\n issues = data.get('issues', data) if isinstance(data, dict) else {}\n target_keys = ['OPS-3', 'OPS-4']\n failures = []\n for key in target_keys:\n issue = issues.get(key)\n if issue is None:\n for k, v in issues.items():\n if isinstance(v, dict) and v.get('key', '').upper() == key.upper():\n issue = v\n break\n if issue is None:\n failures.append(f'{key}: not found in jira_state.json')\n continue\n found_tre900 = False\n # Check labels\n labels = issue.get('labels', [])\n if isinstance(labels, list) and any('TRE-900' in str(l) for l in labels):\n found_tre900 = True\n # Check comments\n if not found_tre900:\n comments = issue.get('comments', [])\n if isinstance(comments, list):\n for comment in comments:\n body = ''\n if isinstance(comment, dict):\n body = str(comment.get('body', '') or comment.get('text', '') or comment.get('content', ''))\n elif isinstance(comment, str):\n body = comment\n if 'TRE-900' in body:\n found_tre900 = True\n break\n # Check custom fields (any field key containing 'custom' or 'exception' or 'code')\n if not found_tre900:\n for field_key, field_val in issue.items():\n if field_key.lower() in ('description', 'summary'):\n continue\n field_str = json.dumps(field_val) if not isinstance(field_val, str) else field_val\n if 'TRE-900' in field_str:\n found_tre900 = True\n break\n if not found_tre900:\n failures.append(f'{key}: TRE-900 not found in labels, comments, or custom fields (description alone does not count)')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'TRE-900 application issues: {failures}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'TRE-900 correctly applied via labels/comments/fields for INTL-2026-0223 and INTL-2026-0228.'}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "7c4a160c-fed8-4ceb-96c1-e1727024a914",
|
||
"sort_order": 7,
|
||
"rubric_text": "In `jira_state.json`, there must be COMP project tickets for both 'Exception Report: International Wire Control Exception – INTL-2026-0223' (or hyphen variant) and 'Exception Report: International Wire Control Exception – INTL-2026-0228' (or hyphen variant), each assigned to diana.walsh. There must be NO COMP ticket whose summary contains 'INTL-2026-0214' or 'INTL-2026-0219'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided.'}\n jira_path = Path(external_services_path) / 'jira_state.json'\n if not jira_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found.'}\n try:\n with open(jira_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse jira_state.json: {e}'}\n issues = data.get('issues', data) if isinstance(data, dict) else {}\n # Search all issues by summary content — don't filter by project key prefix,\n # since models may use COMP-, CA-, COMPLIANCE-, or other schemes.\n comp_issues = [v for v in issues.values() if isinstance(v, dict)]\n def summary_matches(summary, wire_id):\n s = summary.lower()\n return wire_id.lower() in s and ('exception report' in s or 'wire control exception' in s)\n def is_assigned_diana_walsh(issue):\n assignee = issue.get('fields', {}).get('assignee') or issue.get('assignee') or {}\n if isinstance(assignee, dict):\n account_id = str(assignee.get('accountId', '')).lower()\n display_name = str(assignee.get('displayName', '')).lower()\n name = str(assignee.get('name', '')).lower()\n if ('diana' in account_id and 'walsh' in account_id) or \\\n ('diana' in display_name and 'walsh' in display_name) or \\\n ('diana' in name and 'walsh' in name):\n return True\n elif isinstance(assignee, str):\n al = assignee.lower()\n if 'diana' in al and 'walsh' in al:\n return True\n return False\n failures = []\n found_0223 = False\n found_0228 = False\n unwanted_0214 = False\n unwanted_0219 = False\n for issue in comp_issues:\n summary = str(issue.get('fields', {}).get('summary') or issue.get('summary', '')).strip()\n if summary_matches(summary, 'INTL-2026-0214'):\n unwanted_0214 = True\n if summary_matches(summary, 'INTL-2026-0219'):\n unwanted_0219 = True\n if summary_matches(summary, 'INTL-2026-0223'):\n found_0223 = True\n if not is_assigned_diana_walsh(issue):\n failures.append(f'COMP ticket for INTL-2026-0223 not assigned to diana.walsh. Assignee: {issue.get(\"fields\", {}).get(\"assignee\") or issue.get(\"assignee\")}')\n if summary_matches(summary, 'INTL-2026-0228'):\n found_0228 = True\n if not is_assigned_diana_walsh(issue):\n failures.append(f'COMP ticket for INTL-2026-0228 not assigned to diana.walsh. Assignee: {issue.get(\"fields\", {}).get(\"assignee\") or issue.get(\"assignee\")}')\n if not found_0223:\n failures.append('No COMP ticket found for INTL-2026-0223 exception report')\n if not found_0228:\n failures.append('No COMP ticket found for INTL-2026-0228 exception report')\n if unwanted_0214:\n failures.append('COMP ticket for INTL-2026-0214 should NOT exist')\n if unwanted_0219:\n failures.append('COMP ticket for INTL-2026-0219 should NOT exist')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'COMP ticket issues: {failures}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'COMP tickets correctly created for INTL-2026-0223 and INTL-2026-0228, assigned to diana.walsh, and none for 0214/0219.'}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "bc26fbaf-5f13-43f2-997c-353c8a8de92c",
|
||
"sort_order": 8,
|
||
"rubric_text": "In `jira_state.json`, the COMP ticket for INTL-2026-0223 must have a description containing: 'INTL-2026-0223', 'OPS-3', 'OPS-5', '#treasury-wires', 'TRE-900', 'claire.overton' or 'Claire Overton', 'Approved.', and either 'Chief Compliance Officer' or 'CCO'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided.'}\n jira_path = Path(external_services_path) / 'jira_state.json'\n if not jira_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found.'}\n try:\n with open(jira_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse jira_state.json: {e}'}\n issues = data.get('issues', data) if isinstance(data, dict) else {}\n comp_ticket = None\n for k, v in issues.items():\n if not isinstance(v, dict):\n continue\n summary = str(v.get('fields', {}).get('summary') or v.get('summary', ''))\n if 'INTL-2026-0223' in summary and ('exception report' in summary.lower() or 'wire control exception' in summary.lower()):\n comp_ticket = v\n break\n if comp_ticket is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No COMP ticket found for INTL-2026-0223.'}\n desc_raw = comp_ticket.get('fields', {}).get('description') or comp_ticket.get('description', '')\n description = json.dumps(desc_raw) if isinstance(desc_raw, dict) else str(desc_raw)\n required_terms = [\n ('INTL-2026-0223', ['INTL-2026-0223']),\n ('OPS-3', ['OPS-3']),\n ('OPS-5', ['OPS-5']),\n ('#treasury-wires', ['#treasury-wires']),\n ('TRE-900', ['TRE-900']),\n ('claire.overton or Claire Overton', ['claire.overton', 'Claire Overton']),\n ('Approved.', ['Approved.']),\n ('Chief Compliance Officer or CCO', ['Chief Compliance Officer', 'CCO']),\n ]\n missing = []\n for label, terms in required_terms:\n if not any(t in description for t in terms):\n missing.append(label)\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'COMP ticket for INTL-2026-0223 description missing: {missing}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'COMP ticket for INTL-2026-0223 description contains all required elements (wire ID, OPS ticket refs, Slack evidence, TRE-900, approver details, CCO reference).'}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "60018d88-28b8-44ce-b28c-4c2e11a344ee",
|
||
"sort_order": 9,
|
||
"rubric_text": "In `jira_state.json`, the COMP ticket for INTL-2026-0228 must have a description containing: 'INTL-2026-0228', 'OPS-4', 'OPS-5', '#treasury-wires', 'TRE-900', 'claire.overton' or 'Claire Overton', 'Approved.', a missing dual approval indicator (e.g., 'Missing dual approval', 'Missing', or 'no second'), and either 'Chief Compliance Officer' or 'CCO'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided.'}\n jira_path = Path(external_services_path) / 'jira_state.json'\n if not jira_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found.'}\n try:\n with open(jira_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse jira_state.json: {e}'}\n issues = data.get('issues', data) if isinstance(data, dict) else {}\n comp_ticket = None\n for k, v in issues.items():\n if not isinstance(v, dict):\n continue\n summary = str(v.get('fields', {}).get('summary') or v.get('summary', ''))\n if 'INTL-2026-0228' in summary and ('exception report' in summary.lower() or 'wire control exception' in summary.lower()):\n comp_ticket = v\n break\n if comp_ticket is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No COMP ticket found for INTL-2026-0228.'}\n desc_raw = comp_ticket.get('fields', {}).get('description') or comp_ticket.get('description', '')\n description = json.dumps(desc_raw) if isinstance(desc_raw, dict) else str(desc_raw)\n desc_lower = description.lower()\n missing_phrases = ['missing dual approval', 'missing', 'no second', 'no dual', 'dual approval missing', 'absent']\n has_missing_indicator = any(p.lower() in desc_lower for p in missing_phrases)\n required_terms = [\n ('INTL-2026-0228', ['INTL-2026-0228']),\n ('OPS-4', ['OPS-4']),\n ('OPS-5', ['OPS-5']),\n ('#treasury-wires', ['#treasury-wires']),\n ('TRE-900', ['TRE-900']),\n ('claire.overton or Claire Overton', ['claire.overton', 'Claire Overton']),\n ('Approved.', ['Approved.']),\n ('Chief Compliance Officer or CCO', ['Chief Compliance Officer', 'CCO']),\n ]\n missing_fields = []\n for label, terms in required_terms:\n if not any(t in description for t in terms):\n missing_fields.append(label)\n if not has_missing_indicator:\n missing_fields.append('missing dual approval indicator (e.g., Missing dual approval, Missing, no second)')\n if missing_fields:\n return {'pass': False, 'score': 0.0, 'feedback': f'COMP ticket for INTL-2026-0228 description missing: {missing_fields}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'COMP ticket for INTL-2026-0228 description contains all required elements.'}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "e3c590df-0fa7-44f7-9e3a-8a3941627b65",
|
||
"sort_order": 10,
|
||
"rubric_text": "In `slack_data.json`, the #compliance-urgent channel must contain two distinct messages: one for INTL-2026-0223 and one for INTL-2026-0228. Each must include '[Compliance]', 'International wire control exception filed', the respective wire ID, the respective OPS-TRE-WIRE ticket key, and a COMP ticket key pattern. Neither message must contain wire amounts ('623,000' or '312,000') or bank names ('ING Bank' or 'BNP Paribas'). No message in #compliance-urgent must escalate INTL-2026-0214 or INTL-2026-0219 as a wire control exception.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n try:\n with open(slack_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse slack_data.json: {e}'}\n channels = data.get('channels', {})\n comp_urgent_id = None\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict) and ch_info.get('name', '').lower() == 'compliance-urgent':\n comp_urgent_id = ch_id\n break\n if comp_urgent_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#compliance-urgent channel not found in slack_data.json.'}\n messages = data.get('messages', {}).get(comp_urgent_id, [])\n jira_key_pattern = re.compile(r'\\b[A-Z]+-\\d+\\b')\n failures = []\n found_0223 = False\n found_0228 = False\n for msg in messages:\n if not isinstance(msg, dict):\n continue\n text = str(msg.get('text', ''))\n text_lower = text.lower()\n # Check for unwanted escalations\n if 'INTL-2026-0214' in text and ('exception' in text_lower or 'escalat' in text_lower):\n failures.append('Found message escalating INTL-2026-0214 in #compliance-urgent (should not exist)')\n if 'INTL-2026-0219' in text and ('exception' in text_lower or 'escalat' in text_lower):\n failures.append('Found message escalating INTL-2026-0219 in #compliance-urgent (should not exist)')\n # Check for 0223 message\n if 'INTL-2026-0223' in text:\n has_compliance_tag = '[compliance]' in text_lower or '[Compliance]' in text\n has_phrase = 'international wire control exception filed' in text_lower\n has_ops_key = 'OPS-3' in text\n has_comp_key = bool(jira_key_pattern.search(text)) and any(m.upper().startswith('COMP') for m in jira_key_pattern.findall(text))\n no_amount = '623,000' not in text and '623000' not in text\n no_bank = 'ing bank' not in text_lower\n if has_compliance_tag and has_phrase and has_ops_key and has_comp_key and no_amount and no_bank:\n found_0223 = True\n else:\n issues = []\n if not has_compliance_tag: issues.append('missing [Compliance]')\n if not has_phrase: issues.append('missing \"International wire control exception filed\"')\n if not has_ops_key: issues.append('missing OPS-3')\n if not has_comp_key: issues.append('missing COMP ticket key')\n if not no_amount: issues.append('contains wire amount 623,000')\n if not no_bank: issues.append('contains bank name ING Bank')\n if issues:\n failures.append(f'INTL-2026-0223 message issues: {issues}')\n # Check for 0228 message\n if 'INTL-2026-0228' in text:\n has_compliance_tag = '[compliance]' in text_lower or '[Compliance]' in text\n has_phrase = 'international wire control exception filed' in text_lower\n has_ops_key = 'OPS-4' in text\n has_comp_key = bool(jira_key_pattern.search(text)) and any(m.upper().startswith('COMP') for m in jira_key_pattern.findall(text))\n no_amount = '312,000' not in text and '312000' not in text\n no_bank = 'bnp paribas' not in text_lower\n if has_compliance_tag and has_phrase and has_ops_key and has_comp_key and no_amount and no_bank:\n found_0228 = True\n else:\n issues = []\n if not has_compliance_tag: issues.append('missing [Compliance]')\n if not has_phrase: issues.append('missing \"International wire control exception filed\"')\n if not has_ops_key: issues.append('missing OPS-4')\n if not has_comp_key: issues.append('missing COMP ticket key')\n if not no_amount: issues.append('contains wire amount 312,000')\n if not no_bank: issues.append('contains bank name BNP Paribas')\n if issues:\n failures.append(f'INTL-2026-0228 message issues: {issues}')\n if not found_0223:\n failures.append('No valid #compliance-urgent message found for INTL-2026-0223')\n if not found_0228:\n failures.append('No valid #compliance-urgent message found for INTL-2026-0228')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'Slack #compliance-urgent issues: {failures}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'Both INTL-2026-0223 and INTL-2026-0228 compliance-urgent messages verified correctly. No unwanted escalations found.'}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "22f973a2-3243-4911-b491-9df67655de73",
|
||
"sort_order": 11,
|
||
"rubric_text": "In `jira_state.json`, the COMP ticket keys referenced in the #compliance-urgent Slack posts for INTL-2026-0223 and INTL-2026-0228 must correspond to existing issues in jira_state.json.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'external_services_path not provided.'}\n slack_path = Path(external_services_path) / 'slack_data.json'\n jira_path = Path(external_services_path) / 'jira_state.json'\n if not slack_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'slack_data.json not found.'}\n if not jira_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found.'}\n try:\n with open(slack_path, 'r') as f:\n slack_data = json.load(f)\n with open(jira_path, 'r') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Failed to parse files: {e}'}\n jira_issues = jira_data.get('issues', jira_data) if isinstance(jira_data, dict) else {}\n all_jira_keys = set()\n for k, v in jira_issues.items():\n all_jira_keys.add(k.upper())\n if isinstance(v, dict) and v.get('key'):\n all_jira_keys.add(str(v['key']).upper())\n channels = slack_data.get('channels', {})\n comp_urgent_id = None\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict) and ch_info.get('name', '').lower() == 'compliance-urgent':\n comp_urgent_id = ch_id\n break\n if comp_urgent_id is None:\n return {'pass': False, 'score': 0.0, 'feedback': '#compliance-urgent channel not found.'}\n messages = slack_data.get('messages', {}).get(comp_urgent_id, [])\n # Extract key from \"Jira: KEY\" field in B-4 template, falling back to any\n # non-OPS, non-INTL Jira key in the message if the template field is absent.\n jira_field_pattern = re.compile(r'[Jj]ira:\\s*([A-Z][A-Z0-9]*-\\d+)')\n any_key_pattern = re.compile(r'\\b([A-Z][A-Z0-9]*-\\d+)\\b')\n failures = []\n for wire_id in ['INTL-2026-0223', 'INTL-2026-0228']:\n found_msg = False\n for msg in messages:\n if not isinstance(msg, dict):\n continue\n text = str(msg.get('text', ''))\n if wire_id in text and 'international wire control exception filed' in text.lower():\n found_msg = True\n # Prefer the key from the explicit \"Jira: KEY\" field\n comp_keys = jira_field_pattern.findall(text)\n if not comp_keys:\n # Fallback: any key that isn't an OPS- or INTL- pattern\n all_keys = any_key_pattern.findall(text)\n comp_keys = [k for k in all_keys\n if not k.upper().startswith('OPS-')\n and not k.upper().startswith('INTL-')]\n for ck in comp_keys:\n if ck.upper() not in all_jira_keys:\n failures.append(f'Ticket key {ck} referenced in Slack for {wire_id} not found in jira_state.json')\n if not comp_keys:\n failures.append(f'No compliance ticket key found in Slack message for {wire_id}')\n break\n if not found_msg:\n failures.append(f'No qualifying #compliance-urgent message found for {wire_id}')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': f'COMP key cross-reference issues: {failures}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'COMP ticket keys in Slack messages for INTL-2026-0223 and INTL-2026-0228 all exist in jira_state.json.'}",
|
||
"criterion_type": "expected_output"
|
||
}
|
||
]
|