Add Handbook.md benchmark tasks
This commit is contained in:
commit
72f9a65917
1800 files changed
+323589
No files matched your search
@@ -0,0 +1,72 @@
|
||||
[
|
||||
{
|
||||
"id": "3bff4159-1385-44dd-ba6b-1e464d44323f",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In Wire_Population_Q1_2026.xlsx, Wire Population Q1 2026: row 4: INTL-2026-0147 | 312500 | Meridian Reinsurance Partners Ltd. | Deutsche Bank AG, Frankfurt | 2026-03-04 | Q1 2026 | OPS-2 | 2026-03-04, 10:00 AM EST | Karen Stills | VP, Treasury Operations | Approved. | Michael Drury | SVP, Finance & Risk | Approved. | Y | |",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s) # en/em-dash → hyphen\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef val_matches(col_key, actual, expected):\n \"\"\"Timezone-tolerant comparison: slack evidence accepted with or without tz suffix.\"\"\"\n a, e = normalize(actual), normalize(expected)\n if a == e:\n return True\n if col_key == 'slack evidence':\n e_no_tz = re.sub(r'\\s+(est|edt|cst|cdt|mst|mdt|pst|pdt|utc|gmt)$', '', e)\n return a == e_no_tz\n return False\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n spreadsheet_files = (list(ws.glob('**/*.xlsx')) +\n list(ws.glob('**/*.xls')))\n\n if not spreadsheet_files:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No spreadsheet file found in workspace\"}\n\n wire_id = 'INTL-2026-0147'\n\n # Expected values for agent-filled audit columns (pre-normalized)\n expected = {\n 'slack evidence': '2026-03-04, 10:00 am est',\n 'approver 1 name': 'karen stills',\n 'approver 1 title': 'vp, treasury operations',\n 'approver 1 text': 'approved.',\n 'approver 2 name': 'michael drury',\n 'approver 2 title': 'svp, finance & risk',\n 'approver 2 text': 'approved.',\n 'dual auth confirmed (y/n)': 'y',\n 'exception code applied': '',\n }\n\n row_found = False\n checks_passed = []\n checks_failed = []\n\n for fpath in spreadsheet_files:\n try:\n if str(fpath).lower().endswith(('.xlsx', '.xls')):\n import openpyxl\n wb = openpyxl.load_workbook(str(fpath), data_only=True)\n for sheet in wb.sheetnames:\n ws_sheet = wb[sheet]\n all_rows = list(ws_sheet.iter_rows(values_only=True))\n if len(all_rows) < 4:\n continue\n # Row 3 (index 2) = headers, data from row 4 (index 3)\n headers = [str(c).strip() if c else '' for c in all_rows[2]]\n for r in all_rows[3:]:\n cells = [str(c) if c is not None else '' for c in r]\n row_dict = dict(zip(headers, cells))\n wire_val = next(\n (v for k, v in row_dict.items() if normalize(k) == 'wire id'), ''\n )\n if normalize(wire_val) != normalize(wire_id):\n continue\n row_found = True\n for col_key, exp_val in expected.items():\n actual_val = next(\n (v for k, v in row_dict.items() if normalize(k) == col_key), None\n )\n if actual_val is None:\n checks_failed.append(f\"column '{col_key}' not found in headers\")\n continue\n if val_matches(col_key, actual_val, exp_val):\n checks_passed.append(col_key)\n else:\n checks_failed.append(\n f\"'{col_key}': expected '{exp_val}', got '{actual_val}'\"\n )\n except Exception:\n continue\n\n if not row_found:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"{wire_id} not found in any workbook. \"\n f\"Files scanned: {[str(f) for f in spreadsheet_files]}\"}\n\n total = len(expected)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{wire_id}: {passed}/{total} checks passed. \"\n f\"Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"{wire_id}: all {total} audit column checks passed.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "5d5be44e-0560-4c68-9d9c-938557276a78",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In Wire_Population_Q1_2026.xlsx, Wire Population Q1 2026: row 5: INTL-2026-0151 | 87200 | Castlewood Actuarial Services B.V. | ING Bank N.V., Amsterdam | 2026-03-11 | Q1 2026 | OPS-3 | 2026-03-11, 11:00 AM EDT | Karen Stills | VP, Treasury Operations | Approved. | None | None | None | N | TRE-900 High-Risk Exception |",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef val_matches(col_key, actual, expected):\n \"\"\"Timezone-tolerant comparison: slack evidence accepted with or without tz suffix.\"\"\"\n a, e = normalize(actual), normalize(expected)\n if a == e:\n return True\n if col_key == 'slack evidence':\n e_no_tz = re.sub(r'\\s+(est|edt|cst|cdt|mst|mdt|pst|pdt|utc|gmt)$', '', e)\n return a == e_no_tz\n return False\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n spreadsheet_files = (list(ws.glob('**/*.xlsx')) +\n list(ws.glob('**/*.xls')))\n\n if not spreadsheet_files:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No spreadsheet file found in workspace\"}\n\n wire_id = 'INTL-2026-0151'\n\n expected = {\n 'slack evidence': '2026-03-11, 11:00 am edt',\n 'approver 1 name': 'karen stills',\n 'approver 1 title': 'vp, treasury operations',\n 'approver 1 text': 'approved.',\n 'approver 2 name': '',\n 'approver 2 title': '',\n 'approver 2 text': '',\n 'dual auth confirmed (y/n)': 'n',\n 'exception code applied': 'tre-900 high-risk exception',\n }\n\n row_found = False\n checks_passed = []\n checks_failed = []\n\n for fpath in spreadsheet_files:\n try:\n if str(fpath).lower().endswith(('.xlsx', '.xls')):\n import openpyxl\n wb = openpyxl.load_workbook(str(fpath), data_only=True)\n for sheet in wb.sheetnames:\n ws_sheet = wb[sheet]\n all_rows = list(ws_sheet.iter_rows(values_only=True))\n if len(all_rows) < 4:\n continue\n headers = [str(c).strip() if c else '' for c in all_rows[2]]\n for r in all_rows[3:]:\n cells = [str(c) if c is not None else '' for c in r]\n row_dict = dict(zip(headers, cells))\n wire_val = next(\n (v for k, v in row_dict.items() if normalize(k) == 'wire id'), ''\n )\n if normalize(wire_val) != normalize(wire_id):\n continue\n row_found = True\n for col_key, exp_val in expected.items():\n actual_val = next(\n (v for k, v in row_dict.items() if normalize(k) == col_key), None\n )\n if actual_val is None:\n checks_failed.append(f\"column '{col_key}' not found in headers\")\n continue\n if val_matches(col_key, actual_val, exp_val):\n checks_passed.append(col_key)\n else:\n checks_failed.append(\n f\"'{col_key}': expected '{exp_val}', got '{actual_val}'\"\n )\n\n except Exception:\n continue\n\n if not row_found:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"{wire_id} not found in any workbook. \"\n f\"Files scanned: {[str(f) for f in spreadsheet_files]}\"}\n\n total = len(expected)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{wire_id}: {passed}/{total} checks passed. \"\n f\"Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"{wire_id}: all {total} audit column checks passed.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "09539af3-6784-4fc4-8ea5-943d5ab0985b",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In Wire_Population_Q1_2026.xlsx, Wire Population Q1 2026: row 6: INTL-2026-0158 | 204750 | Nordic Claims Recovery AS | DNB Bank ASA, Oslo | 2026-03-19 | Q1 2026 | OPS-4 | 2026-03-19, 11:00 AM EDT | Karen Stills | VP, Treasury Operations | Approved. | Brett Connors | Senior Manager, Treasury Operations | Approved. | N | TRE-900 High-Risk Exception |",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef val_matches(col_key, actual, expected):\n \"\"\"Timezone-tolerant comparison: slack evidence accepted with or without tz suffix.\"\"\"\n a, e = normalize(actual), normalize(expected)\n if a == e:\n return True\n if col_key == 'slack evidence':\n e_no_tz = re.sub(r'\\s+(est|edt|cst|cdt|mst|mdt|pst|pdt|utc|gmt)$', '', e)\n return a == e_no_tz\n return False\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n spreadsheet_files = (list(ws.glob('**/*.xlsx')) +\n list(ws.glob('**/*.xls')))\n\n if not spreadsheet_files:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No spreadsheet file found in workspace\"}\n\n wire_id = 'INTL-2026-0158'\n\n expected = {\n 'slack evidence': '2026-03-19, 11:00 am edt',\n 'approver 1 name': 'karen stills',\n 'approver 1 title': 'vp, treasury operations',\n 'approver 1 text': 'approved.',\n 'approver 2 name': 'brett connors',\n 'approver 2 title': 'senior manager, treasury operations',\n 'approver 2 text': 'approved.',\n 'dual auth confirmed (y/n)': 'n',\n 'exception code applied': 'tre-900 high-risk exception',\n }\n\n row_found = False\n checks_passed = []\n checks_failed = []\n\n for fpath in spreadsheet_files:\n try:\n if str(fpath).lower().endswith(('.xlsx', '.xls')):\n import openpyxl\n wb = openpyxl.load_workbook(str(fpath), data_only=True)\n for sheet in wb.sheetnames:\n ws_sheet = wb[sheet]\n all_rows = list(ws_sheet.iter_rows(values_only=True))\n if len(all_rows) < 4:\n continue\n headers = [str(c).strip() if c else '' for c in all_rows[2]]\n for r in all_rows[3:]:\n cells = [str(c) if c is not None else '' for c in r]\n row_dict = dict(zip(headers, cells))\n wire_val = next(\n (v for k, v in row_dict.items() if normalize(k) == 'wire id'), ''\n )\n if normalize(wire_val) != normalize(wire_id):\n continue\n row_found = True\n for col_key, exp_val in expected.items():\n actual_val = next(\n (v for k, v in row_dict.items() if normalize(k) == col_key), None\n )\n if actual_val is None:\n checks_failed.append(f\"column '{col_key}' not found in headers\")\n continue\n if val_matches(col_key, actual_val, exp_val):\n checks_passed.append(col_key)\n else:\n checks_failed.append(\n f\"'{col_key}': expected '{exp_val}', got '{actual_val}'\"\n )\n\n except Exception:\n continue\n\n if not row_found:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"{wire_id} not found in any workbook. \"\n f\"Files scanned: {[str(f) for f in spreadsheet_files]}\"}\n\n total = len(expected)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{wire_id}: {passed}/{total} checks passed. \"\n f\"Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"{wire_id}: all {total} audit column checks passed.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e60a6566-2dc9-4d44-a89b-3489886049fa",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In Wire_Population_Q1_2026.xlsx, Wire Population Q1 2026: row 7: INTL-2026-0163 | 519000 | Avondale Risk Consulting Ltd. | Barclays Bank PLC, London | 2026-03-27 | Q1 2026 | OPS-5 | 2026-03-27, 11:00 AM EDT | Karen Stills | VP, Treasury Operations | Approved. | Priya Mehta | AVP, Enterprise Risk | Approved. | Y | |",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef val_matches(col_key, actual, expected):\n \"\"\"Timezone-tolerant comparison: slack evidence accepted with or without tz suffix.\"\"\"\n a, e = normalize(actual), normalize(expected)\n if a == e:\n return True\n if col_key == 'slack evidence':\n e_no_tz = re.sub(r'\\s+(est|edt|cst|cdt|mst|mdt|pst|pdt|utc|gmt)$', '', e)\n return a == e_no_tz\n return False\n\ndef verify(workspace_path, external_services_path=None):\n ws = Path(workspace_path)\n spreadsheet_files = (list(ws.glob('**/*.xlsx')) +\n list(ws.glob('**/*.xls')))\n\n if not spreadsheet_files:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"No spreadsheet file found in workspace\"}\n\n wire_id = 'INTL-2026-0163'\n\n expected = {\n 'slack evidence': '2026-03-27, 11:00 am edt',\n 'approver 1 name': 'karen stills',\n 'approver 1 title': 'vp, treasury operations',\n 'approver 1 text': 'approved.',\n 'approver 2 name': 'priya mehta',\n 'approver 2 title': 'avp, enterprise risk',\n 'approver 2 text': 'approved.',\n 'dual auth confirmed (y/n)': 'y',\n 'exception code applied': '',\n }\n\n row_found = False\n checks_passed = []\n checks_failed = []\n\n for fpath in spreadsheet_files:\n try:\n if str(fpath).lower().endswith(('.xlsx', '.xls')):\n import openpyxl\n wb = openpyxl.load_workbook(str(fpath), data_only=True)\n for sheet in wb.sheetnames:\n ws_sheet = wb[sheet]\n all_rows = list(ws_sheet.iter_rows(values_only=True))\n if len(all_rows) < 4:\n continue\n headers = [str(c).strip() if c else '' for c in all_rows[2]]\n for r in all_rows[3:]:\n cells = [str(c) if c is not None else '' for c in r]\n row_dict = dict(zip(headers, cells))\n wire_val = next(\n (v for k, v in row_dict.items() if normalize(k) == 'wire id'), ''\n )\n if normalize(wire_val) != normalize(wire_id):\n continue\n row_found = True\n for col_key, exp_val in expected.items():\n actual_val = next(\n (v for k, v in row_dict.items() if normalize(k) == col_key), None\n )\n if actual_val is None:\n checks_failed.append(f\"column '{col_key}' not found in headers\")\n continue\n if val_matches(col_key, actual_val, exp_val):\n checks_passed.append(col_key)\n else:\n checks_failed.append(\n f\"'{col_key}': expected '{exp_val}', got '{actual_val}'\"\n )\n\n except Exception:\n continue\n\n if not row_found:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"{wire_id} not found in any workbook. \"\n f\"Files scanned: {[str(f) for f in spreadsheet_files]}\"}\n\n total = len(expected)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{wire_id}: {passed}/{total} checks passed. \"\n f\"Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"{wire_id}: all {total} audit column checks passed.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "3ef4031c-e646-4b97-b548-9c4ec5e5e650",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `jira_state.json`, issues OPS-2 and OPS-5 must each have a comment whose body contains the exact text `status=verified`, and must not have a TRE-900 escalation comment",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\u2013\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\n\ndef extract_text(node):\n \"\"\"Recursively extract plain text from a Jira doc object or return string as-is.\"\"\"\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(child) for child in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(item) for item in node)\n return str(node)\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path, encoding='utf-8') as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n all_comments = data.get(\"comments\", {})\n\n target_keys = ['OPS-2', 'OPS-5']\n # SOP 3.3 Happy path: dual-auth confirmation is recorded by adding a comment\n # that includes the exact token below. (A status change is not representable in\n # the Jira workflow, so verification lives in a comment.) Matched exactly and\n # case-sensitively so the token is stable for any downstream consumer.\n expected_token = 'status=verified'\n checks_passed = []\n checks_failed = []\n\n for key in target_keys:\n issue = issues.get(key)\n if issue is None:\n checks_failed.append(f\"{key}: not found in jira_state.json\")\n checks_failed.append(f\"{key}: verification-comment check skipped (issue missing)\")\n checks_failed.append(f\"{key}: not-escalated check skipped (issue missing)\")\n continue\n\n comments = all_comments.get(key, [])\n comments = comments if isinstance(comments, list) else []\n bodies = [extract_text(c.get('body', '') if isinstance(c, dict) else c) for c in comments]\n\n # Check 1: a comment carries the exact verification token 'status=verified'\n if any(expected_token in b for b in bodies):\n checks_passed.append(f\"{key}: has comment containing '{expected_token}'\")\n else:\n checks_failed.append(\n f\"{key}: no comment contains exact token '{expected_token}'. Comments: {bodies}\")\n\n # Check 2: a verified wire must NOT be escalated (no TRE-900 exception comment)\n if not any('tre-900' in normalize(b) for b in bodies):\n checks_passed.append(f\"{key}: not escalated (no TRE-900 comment)\")\n else:\n checks_failed.append(\n f\"{key}: has a TRE-900 escalation comment; a verified wire must not be escalated. \"\n f\"Comments: {bodies}\")\n\n total = len(target_keys) * 2 # verification-comment + not-escalated, per wire\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{passed}/{total} checks passed. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"Both OPS-2 and OPS-5 \"\n \"carry a 'status=verified' comment and were not escalated.\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e48deaee-b488-4278-9893-899ac8afac57",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `jira_state.json`, Issue OPS-3 has status \"Open\" and exactly 1 comment: \"TRE-900 High-Risk Exception\" and Issue OPS-4 has status \"Open\" and exactly 1 comment: \"TRE-900 High-Risk Exception\"",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef extract_text(node):\n \"\"\"Recursively extract plain text from a Jira doc object or return string as-is.\"\"\"\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(child) for child in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(item) for item in node)\n return str(node)\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path, encoding='utf-8') as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n all_comments = data.get(\"comments\", {})\n\n target_keys = ['OPS-3', 'OPS-4']\n expected_comment = 'TRE-900 High-Risk Exception'\n checks_passed = []\n checks_failed = []\n\n for key in target_keys:\n issue = issues.get(key)\n if issue is None:\n checks_failed.append(f\"{key}: not found in jira_state.json\")\n checks_failed.append(f\"{key}: comment count check skipped (issue missing)\")\n checks_failed.append(f\"{key}: comment text check skipped (issue missing)\")\n continue\n\n # Status check\n status = issue.get('fields', {}).get('status', {}).get('name', '')\n if status.lower() == 'open':\n checks_passed.append(f\"{key}: status='{status}'\")\n else:\n checks_failed.append(f\"{key}: status='{status}', expected 'Open'\")\n\n # Comment count check\n comments = all_comments.get(key, [])\n comments = comments if isinstance(comments, list) else []\n if len(comments) == 1:\n checks_passed.append(f\"{key}: comment count=1\")\n else:\n checks_failed.append(f\"{key}: {len(comments)} comment(s), expected exactly 1\")\n\n # Comment text check\n if len(comments) == 1:\n raw = comments[0].get('body', '') if isinstance(comments[0], dict) else comments[0]\n body = extract_text(raw)\n if normalize(body) == normalize(expected_comment):\n checks_passed.append(f\"{key}: comment text matches\")\n else:\n checks_failed.append(\n f\"{key}: comment text expected '{expected_comment}', got '{body}'\"\n )\n else:\n checks_failed.append(f\"{key}: comment text check skipped (wrong count)\")\n\n total = len(target_keys) * 3 # status + count + text per wire\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{passed}/{total} checks passed. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Both OPS-3 and OPS-4 \"\n f\"are Open with exactly 1 comment: '{expected_comment}'.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "88394811-143f-4668-8c1b-ce9fc48da63f",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In slack_data.json: \n- In C002 a message with text: Compliance International wire control exception filed — Jira: OPS-7 | Wire ID: INTL-2026-0151 | OPS-3 | Junior Operations Analyst\n- In C002 a message with text: Compliance International wire control exception filed — Jira: OPS-8 | Wire ID: INTL-2026-0158 | OPS-4 | Junior Operations Analyst",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path, encoding='utf-8') as f:\n data = json.load(f)\n\n # Resolve #compliance-urgent by name\n channels = data.get(\"channels\", {})\n compliance_id = next(\n (cid for cid, info in channels.items()\n if isinstance(info, dict) and info.get('name') == 'compliance-urgent'),\n None\n )\n if compliance_id is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"Channel #compliance-urgent not found in slack_data.json\"}\n\n channel_msgs = data.get(\"messages\", {}).get(compliance_id, [])\n if not isinstance(channel_msgs, list):\n channel_msgs = []\n\n # Gate: exactly 2 messages required\n if len(channel_msgs) != 2:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"#compliance-urgent has {len(channel_msgs)} message(s), expected exactly 2\"}\n\n expected_texts = [\n \"Compliance International wire control exception filed - Jira: OPS-7 | Wire ID: INTL-2026-0151 | OPS-3 | Junior Operations Analyst\",\n \"Compliance International wire control exception filed - Jira: OPS-8 | Wire ID: INTL-2026-0158 | OPS-4 | Junior Operations Analyst\",\n ]\n\n actual_texts = [str(m.get('text', '')) for m in channel_msgs]\n checks_passed = []\n checks_failed = []\n\n for exp in expected_texts:\n exp_norm = normalize(exp)\n match = next((a for a in actual_texts if normalize(a) == exp_norm), None)\n if match is not None:\n checks_passed.append(exp)\n else:\n checks_failed.append(\n f\"Expected message not found: '{exp}'. \"\n f\"Actual messages: {actual_texts}\"\n )\n\n total = len(expected_texts)\n passed = len(checks_passed)\n score = round(passed / total, 4)\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"{passed}/{total} messages matched. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"Both expected #compliance-urgent messages found with exact content.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776028557759",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `jira_state.json`: Exactly 1 issue exists with the following properties:\n- **Summary:** `Exception Report: International Wire Control Exception – INTL-2026-0151`\n- **Project:** `OPS`\n- **Key:** `OPS-7`\n- **Type:** `Task`\n- **Priority:** `High`\n- **Status:** `Open`\n- **Assignee:** `diana.walsh`\n- **Description** must contain all of the following:\n - `Wire ID: INTL-2026-0151`\n - `OPS-TRE-WIRE issue: OPS-3`\n - `Population source: International wires population workbook from OPS-6 (Wire_Population_Q1_2026.xlsx)`\n - `Slack evidence: #treasury-wires | Thread timestamp: 2026-03-11, 11:00 AM EDT`\n - `Approver 1: Karen Stills | VP, Treasury Operations | Approved.`\n - `Approver 2: Missing`\n - `Exception code applied on wire issue: TRE-900 High-Risk Exception`\n - `Action requested: Chief Compliance Officer (per Compliance-Alerts board pinned issue) to determine corrective action and whether wire may proceed.`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef extract_text(node):\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(c) for c in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(i) for i in node)\n return str(node)\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path, encoding='utf-8') as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n issue = issues.get(\"OPS-7\")\n if issue is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Issue OPS-7 not found. Keys present: {list(issues.keys())}\"}\n\n fields = issue.get(\"fields\", {})\n checks_passed = []\n checks_failed = []\n\n # Metadata checks\n meta = {\n \"summary\": (fields.get(\"summary\", \"\"),\n \"Exception Report: International Wire Control Exception – INTL-2026-0151\"),\n \"issuetype\": (fields.get(\"issuetype\", {}).get(\"name\", \"\"), \"task\"),\n \"priority\": (fields.get(\"priority\", {}).get(\"name\", \"\"), \"high\"),\n \"project\": (fields.get(\"project\", {}).get(\"key\", \"\"), \"ops\"),\n }\n \n # Status: accept Open or To Do\n status_actual = normalize(fields.get(\"status\", {}).get(\"name\", \"\"))\n if status_actual in (\"open\", \"to do\"):\n checks_passed.append(\"status\")\n else:\n checks_failed.append(\n f\"status: expected 'Open' or 'To Do', got '{fields.get('status', {}).get('name', '')}'\"\n )\n\n for field, (actual, expected) in meta.items():\n if normalize(actual) == normalize(expected):\n checks_passed.append(field)\n else:\n checks_failed.append(f\"{field}: expected '{expected}', got '{actual}'\")\n\n # Assignee (may be string username or user object)\n assignee_raw = fields.get(\"assignee\", \"\")\n if isinstance(assignee_raw, dict):\n assignee = assignee_raw.get(\"name\", assignee_raw.get(\"accountId\",\n assignee_raw.get(\"emailAddress\", \"\")))\n else:\n assignee = str(assignee_raw)\n if normalize(assignee) == \"diana.walsh\":\n checks_passed.append(\"assignee\")\n else:\n checks_failed.append(f\"assignee: expected 'diana.walsh', got '{assignee}'\")\n\n # Description content\n desc_text = extract_text(fields.get(\"description\", \"\"))\n desc_norm = normalize(desc_text)\n\n description_lines = [\n \"Wire ID: INTL-2026-0151\",\n \"OPS-TRE-WIRE issue: OPS-3\",\n \"Population source: International wires population workbook from OPS-6 (Wire_Population_Q1_2026.xlsx)\",\n \"Slack evidence: #treasury-wires | Thread timestamp: 2026-03-11, 11:00 AM EDT\",\n \"Approver 1: Karen Stills | VP, Treasury Operations | Approved.\",\n \"Approver 2: Missing\",\n \"Exception code applied on wire issue: TRE-900 High-Risk Exception\",\n \"Action requested: Chief Compliance Officer (per Compliance-Alerts board pinned issue) to determine corrective action and whether wire may proceed.\",\n ]\n for line in description_lines:\n if normalize(line) in desc_norm:\n checks_passed.append(f\"desc:'{line[:50]}'\")\n else:\n checks_failed.append(f\"description missing: '{line}'\")\n\n total = len(checks_passed) + len(checks_failed)\n passed = len(checks_passed)\n score = round(passed / total, 4) if total > 0 else 0.0\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"OPS-7: {passed}/{total} checks passed. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"OPS-7: all {total} checks passed.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776028592601",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `jira_state.json`: Exactly 1 issue exists with the following properties:\n- **Summary:** `Exception Report: International Wire Control Exception – INTL-2026-0158`\n- **Project:** `OPS`\n- **Key:** `OPS-8`\n- **Type:** `Task`\n- **Priority:** `High`\n- **Status:** `Open`\n- **Assignee:** `diana.walsh`\n- **Description** must contain all of the following:\n - `Wire ID: INTL-2026-0158`\n - `OPS-TRE-WIRE issue: OPS-4`\n - `Population source: International wires population workbook from OPS-6 (Wire_Population_Q1_2026.xlsx)`\n - `Slack evidence: #treasury-wires | Thread timestamp: 2026-03-19, 11:00 AM EDT`\n - `Approver 1: Karen Stills | VP, Treasury Operations | Approved.`\n - `Approver 2: Brett Connors | Senior Manager, Treasury Operations | Approved. | title does not meet threshold`\n - `Exception code applied on wire issue: TRE-900 High-Risk Exception`\n - `Action requested: Chief Compliance Officer (per Compliance-Alerts board pinned issue) to determine corrective action and whether wire may proceed.`",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef extract_text(node):\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n if node.get('type') == 'text':\n return node.get('text', '')\n return ' '.join(extract_text(c) for c in node.get('content', []))\n if isinstance(node, list):\n return ' '.join(extract_text(i) for i in node)\n return str(node)\n\ndef normalize(s):\n s = str(s).strip()\n s = re.sub(r'[\\u2013\\u2014]', '-', s)\n s = re.sub(r'\\s+', ' ', s)\n return s.lower()\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check Jira state\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path, encoding='utf-8') as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {})\n issue = issues.get(\"OPS-8\")\n if issue is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Issue OPS-8 not found. Keys present: {list(issues.keys())}\"}\n\n fields = issue.get(\"fields\", {})\n checks_passed = []\n checks_failed = []\n\n # Metadata checks\n meta = {\n \"summary\": (fields.get(\"summary\", \"\"),\n \"Exception Report: International Wire Control Exception – INTL-2026-0158\"),\n \"issuetype\": (fields.get(\"issuetype\", {}).get(\"name\", \"\"), \"task\"),\n \"priority\": (fields.get(\"priority\", {}).get(\"name\", \"\"), \"high\"),\n \"project\": (fields.get(\"project\", {}).get(\"key\", \"\"), \"ops\"),\n }\n \n # Status: accept Open or To Do\n status_actual = normalize(fields.get(\"status\", {}).get(\"name\", \"\"))\n if status_actual in (\"open\", \"to do\"):\n checks_passed.append(\"status\")\n else:\n checks_failed.append(\n f\"status: expected 'Open' or 'To Do', got '{fields.get('status', {}).get('name', '')}'\"\n )\n\n for field, (actual, expected) in meta.items():\n if normalize(actual) == normalize(expected):\n checks_passed.append(field)\n else:\n checks_failed.append(f\"{field}: expected '{expected}', got '{actual}'\")\n\n # Assignee (may be string username or user object)\n assignee_raw = fields.get(\"assignee\", \"\")\n if isinstance(assignee_raw, dict):\n assignee = assignee_raw.get(\"name\", assignee_raw.get(\"accountId\",\n assignee_raw.get(\"emailAddress\", \"\")))\n else:\n assignee = str(assignee_raw)\n if normalize(assignee) == \"diana.walsh\":\n checks_passed.append(\"assignee\")\n else:\n checks_failed.append(f\"assignee: expected 'diana.walsh', got '{assignee}'\")\n\n # Description content\n desc_text = extract_text(fields.get(\"description\", \"\"))\n desc_norm = normalize(desc_text)\n\n # description_lines:\n description_lines = [\n \"Wire ID: INTL-2026-0158\",\n \"OPS-TRE-WIRE issue: OPS-4\",\n \"Population source: International wires population workbook from OPS-6 (Wire_Population_Q1_2026.xlsx)\",\n \"Slack evidence: #treasury-wires | Thread timestamp: 2026-03-19, 11:00 AM EDT\",\n \"Approver 1: Karen Stills | VP, Treasury Operations | Approved.\",\n \"Approver 2: Brett Connors | Senior Manager, Treasury Operations | Approved. | title does not meet threshold\",\n \"Exception code applied on wire issue: TRE-900 High-Risk Exception\",\n \"Action requested: Chief Compliance Officer (per Compliance-Alerts board pinned issue) to determine corrective action and whether wire may proceed.\",\n ]\n\n for line in description_lines:\n if normalize(line) in desc_norm:\n checks_passed.append(f\"desc:'{line[:50]}'\")\n else:\n checks_failed.append(f\"description missing: '{line}'\")\n\n total = len(checks_passed) + len(checks_failed)\n passed = len(checks_passed)\n score = round(passed / total, 4) if total > 0 else 0.0\n\n if checks_failed:\n return {\"pass\": False, \"score\": score,\n \"feedback\": f\"OPS-8: {passed}/{total} checks passed. Failures: {checks_failed}\"}\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"OPS-8: all {total} checks passed.\"}\n",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1776028631592",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "1. In `jira_state.json`: exactly 8 total issues.\n2. In `slack_data.json`: exactly 11 messages in channel C001 (treasury-wires) and exactly 2 messages in channel C002 (compliance-urgent).\n3. In `mailbox.json`: exactly 0 emails in Sent, Draft, and Trash folders.\n4. In `calendar_data.json`: exactly 3 events.\n5. In `Wire_Population_Q1_2026.xlsx`, sheet \"Wire Population Q1 2026\": exactly 7 rows (3 header rows + 4 data rows).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"external_services_path is None; cannot check external service files\"}\n\n feedback_parts = []\n all_pass = True\n total_checks = 5\n passed_checks = 0\n\n # ---- Check 1: jira_state.json has exactly 8 issues ----\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n feedback_parts.append(\"FAIL [Check 1]: jira_state.json not found.\")\n all_pass = False\n else:\n try:\n with open(jira_path, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n issues = jira_data.get(\"issues\", {})\n issue_count = len(issues)\n if issue_count == 8:\n feedback_parts.append(\"PASS [Check 1]: jira_state.json has exactly 8 issues.\")\n passed_checks += 1\n else:\n feedback_parts.append(f\"FAIL [Check 1]: jira_state.json has {issue_count} issues, expected 8. \"\n f\"Keys: {list(issues.keys())}\")\n all_pass = False\n except Exception as e:\n feedback_parts.append(f\"FAIL [Check 1]: Error reading jira_state.json: {e}\")\n all_pass = False\n\n # ---- Check 2: slack_data.json channel message counts ----\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n feedback_parts.append(\"FAIL [Check 2]: slack_data.json not found.\")\n all_pass = False\n else:\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n slack_data = json.load(f)\n\n messages = slack_data.get(\"messages\", {})\n channels = slack_data.get(\"channels\", {})\n\n treasury_id = next(\n (cid for cid, info in channels.items()\n if isinstance(info, dict) and info.get(\"name\") == \"treasury-wires\"), None\n )\n compliance_id = next(\n (cid for cid, info in channels.items()\n if isinstance(info, dict) and info.get(\"name\") == \"compliance-urgent\"), None\n )\n\n c001_count = len(messages.get(treasury_id, [])) if treasury_id else None\n c002_count = len(messages.get(compliance_id, [])) if compliance_id else None\n\n c2_pass = True\n if c001_count is None:\n feedback_parts.append(\"FAIL [Check 2a]: treasury-wires channel not found.\")\n all_pass = False; c2_pass = False\n elif c001_count != 11:\n feedback_parts.append(f\"FAIL [Check 2a]: treasury-wires has {c001_count} messages, expected 11.\")\n all_pass = False; c2_pass = False\n else:\n feedback_parts.append(\"PASS [Check 2a]: treasury-wires has exactly 11 messages.\")\n\n if c002_count is None:\n feedback_parts.append(\"FAIL [Check 2b]: compliance-urgent channel not found.\")\n all_pass = False; c2_pass = False\n elif c002_count != 2:\n feedback_parts.append(f\"FAIL [Check 2b]: compliance-urgent has {c002_count} messages, expected 2.\")\n all_pass = False; c2_pass = False\n else:\n feedback_parts.append(\"PASS [Check 2b]: compliance-urgent has exactly 2 messages.\")\n\n if c2_pass:\n passed_checks += 1\n except Exception as e:\n feedback_parts.append(f\"FAIL [Check 2]: Error reading slack_data.json: {e}\")\n all_pass = False\n\n # ---- Check 3: mailbox.json has 0 emails in Sent, Draft, Trash ----\n mail_path = Path(external_services_path) / \"mailbox.json\"\n if not mail_path.exists():\n feedback_parts.append(\"FAIL [Check 3]: mailbox.json not found.\")\n all_pass = False\n else:\n try:\n with open(mail_path, \"r\", encoding=\"utf-8\") as f:\n mail_data = json.load(f)\n emails = mail_data.get(\"emails\", [])\n sent_count = sum(1 for e in emails if str(e.get(\"folder\", \"\")).lower() == \"sent\")\n draft_count = sum(1 for e in emails if str(e.get(\"folder\", \"\")).lower() in (\"draft\", \"drafts\"))\n trash_count = sum(1 for e in emails if str(e.get(\"folder\", \"\")).lower() == \"trash\")\n if sent_count == 0 and draft_count == 0 and trash_count == 0:\n feedback_parts.append(\"PASS [Check 3]: mailbox.json has 0 Sent, 0 Draft, 0 Trash emails.\")\n passed_checks += 1\n else:\n feedback_parts.append(f\"FAIL [Check 3]: Sent={sent_count}, Draft={draft_count}, \"\n Line truncated
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
It is April 10, 2026, 09:00 AM EDT. You are a junior operations analyst at Vanguard Shield Mutual. Complete the Q1 '26 international wire transfer dual-authorization audit, strictly following SOP. Open the Jira issue and the wire pop workbook identify any wire that lacks dual authorization and follow the escalation procedure for each. Follow the §3.3 exactly, record each outcome in the workbook.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user