Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge committed 2026-06-29 10:08:59 -07:00
commit 72f9a65917
1800 files changed
+323589

No files matched your search

@@ -0,0 +1,86 @@
[
{
"id": "cd482ae3-5a98-473d-944b-90077a455e30",
"sort_order": 0,
"rubric_text": "File `Suspense_Reconciliation_March2026.xlsx` must exist in the workspace and contain columns: Support Found, Support Location, Conflict, Status, and Resolution Notes.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip() if cell.value else '' for cell in ws[1]]\n headers_lower = [h.lower() for h in headers]\n required_cols = ['support found', 'support location', 'conflict', 'status', 'resolution notes']\n missing = [col for col in required_cols if col not in headers_lower]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns: {missing}. Found columns: {headers}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'File exists and contains all required columns: {required_cols}'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading file: {e}'}",
"criterion_type": "expected_output"
},
{
"id": "294d7b7f-70c3-46e0-993b-71b9f0cecf2c",
"sort_order": 1,
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, cleared items OPS-2 ($3,250), OPS-3 ($7,800), OPS-7 ($2,100), and OPS-8 ($15,000) must each show Status=FIN-100, Support Found=Y, Conflict=N, and non-empty Resolution Notes.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n def col_idx(name):\n for i, h in enumerate(headers):\n if name.lower() in h:\n return i\n return None\n id_col = col_idx('susp') if col_idx('susp') is not None else col_idx('id') if col_idx('id') is not None else 0\n # Try to find transaction ID column\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or (i > 0 and 'support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h == 'support found':\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [status_col, support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find required columns. Headers: {headers}'}\n target_ids = {'OPS-2', 'OPS-3', 'OPS-7', 'OPS-8'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n status = str(row[status_col]).strip() if row[status_col] else ''\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip() if row[notes_col] else ''\n if 'FIN-100' not in status.upper() and status.upper() != 'FIN-100':\n failures.append(f'{tid}: Status={status} (expected FIN-100)')\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'{tid}: Support Found={sf} (expected Y)')\n if conflict not in ('N', 'NO', 'FALSE', '0'):\n failures.append(f'{tid}: Conflict={conflict} (expected N)')\n if not notes or notes.lower() == 'none':\n failures.append(f'{tid}: Resolution Notes is empty')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': f'All cleared items (OPS-2, OPS-3, OPS-7, OPS-8) show Status=FIN-100, Support Found=Y, Conflict=N, and non-empty Resolution Notes'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
"criterion_type": "expected_output"
},
{
"id": "731b9a6e-3474-439c-95c7-5855d1a54007",
"sort_order": 2,
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-3 ($7,800) and OPS-8 ($15,000) Resolution Notes must reference a Slack approval message.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if notes_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Resolution Notes column. Headers: {headers}'}\n target_ids = {'OPS-3', 'OPS-8'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n slack_keywords = ['slack', 'approval', 'approved', '#finance', 'channel', 'message']\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if not any(kw in notes for kw in slack_keywords):\n failures.append(f'{tid}: Resolution Notes does not reference Slack approval. Notes: \"{notes}\"')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'OPS-3 and OPS-8 Resolution Notes reference Slack approval message'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
"criterion_type": "expected_output"
},
{
"id": "753ac24a-0b95-46ad-9b11-a3a2dd24d630",
"sort_order": 3,
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, exception items OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, and OPS-12 must each show Status=EXCEPTION and must NOT show Status=FIN-100.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n if status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Status column. Headers: {headers}'}\n target_ids = {'OPS-4', 'OPS-5', 'OPS-6', 'OPS-9', 'OPS-10', 'OPS-11', 'OPS-12'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n status = str(row[status_col]).strip().upper() if row[status_col] else ''\n if 'EXCEPTION' not in status:\n failures.append(f'{tid}: Status={status} (expected EXCEPTION)')\n if 'FIN-100' in status:\n failures.append(f'{tid}: Status={status} must NOT be FIN-100')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'All exception items (OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, OPS-12) show Status=EXCEPTION'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
"criterion_type": "expected_output"
},
{
"id": "e23f7481-1554-4d6d-9055-770e06854a17",
"sort_order": 4,
"rubric_text": "In file Suspense_Reconciliation_March2026.xlsx, OPS-4 must show Support Found=Y, Conflict=N, and Resolution Notes indicating missing Slack approval. OPS-5 must show Support Found=N, Conflict=N, and Resolution Notes indicating missing PDF. OPS-11 must show Support Found=Y, Conflict=N, and Resolution Notes indicating no manager approval found in #finance-approvals. OPS-12 must show Support Found=Y, Conflict=N, and Resolution Notes indicating approval must be confirmed by a manager, not a junior analyst.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file - be flexible\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n candidates = list(Path(workspace_path).glob('*Suspense*Reconciliation*March*2026*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*reconciliation*march*2026*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*Suspense*March*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*march*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*Suspense*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*.xlsx'))\n if candidates:\n xlsx_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = None\n for sheet_name in wb.sheetnames:\n candidate_ws = wb[sheet_name]\n first_row = list(candidate_ws.iter_rows(min_row=1, max_row=1, values_only=True))\n if not first_row:\n continue\n candidate_headers = [str(cell).strip().lower() if cell else '' for cell in first_row[0]]\n has_support = any('support' in h for h in candidate_headers)\n has_conflict = any('conflict' in h for h in candidate_headers)\n has_notes = any('resolution' in h or 'notes' in h for h in candidate_headers)\n if has_support and has_conflict and has_notes:\n ws = candidate_ws\n break\n if ws is None:\n ws = wb.active\n\n first_row = list(ws.iter_rows(min_row=1, max_row=1, values_only=True))\n if not first_row or not first_row[0]:\n return {'pass': False, 'score': 0.0, 'feedback': 'No headers found in first row'}\n headers = [str(cell).strip().lower() if cell else '' for cell in first_row[0]]\n\n id_col = None\n for i, h in enumerate(headers):\n if 'susp' in h or h == 'id' or 'item id' in h or 'item_id' in h:\n id_col = i\n break\n if id_col is None:\n for i, h in enumerate(headers):\n if 'item' in h or 'transaction' in h or 'entry' in h or 'ref' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if 'support' in h and 'location' not in h and 'loc' not in h:\n support_found_col = i\n break\n\n support_location_col = None\n for i, h in enumerate(headers):\n if 'support location' in h or ('support' in h and 'location' in h) or ('support' in h and 'loc' in h):\n support_location_col = i\n break\n\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n\n status_col = None\n for i, h in enumerate(headers):\n if h == 'status' or 'status' in h:\n status_col = i\n break\n\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h and 'note' in h:\n notes_col = i\n break\n if notes_col is None:\n for i, h in enumerate(headers):\n if 'resolution' in h:\n notes_col = i\n break\n if notes_col is None:\n for i, h in enumerate(headers):\n if 'notes' in h or 'note' in h:\n notes_col = i\n break\n if notes_col is None:\n for i, h in enumerate(headers):\n if 'comment' in h or 'remark' in h:\n notes_col = i\n break\n\n missing_cols = []\n if support_found_col is None:\n missing_cols.append('Support Found')\n if conflict_col is None:\n missing_cols.append('Conflict')\n if notes_col is None:\n missing_cols.append('Resolution Notes')\n if missing_cols:\n return {'pass': False, 'score': 0.0, 'feedback': f'MissingLine truncated
"criterion_type": "expected_output"
},
{
"id": "8c35192a-a420-4647-9f7b-a056ddad0a04",
"sort_order": 5,
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-6 must show Support Found=Y, Conflict=Y, and Resolution Notes documenting both GL codes (Invoice: 7100-Capital-Equipment and Excel: 8100-IT-Equipment). OPS-10 must show Support Found=Y, Conflict=Y, and Resolution Notes documenting both GL codes (Invoice: 6200-Office-Supplies and Excel: 6200-Office-Exp).",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h.startswith('support'):\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns. Headers: {headers}'}\n rows = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in ('OPS-6', 'OPS-10'):\n rows[row_id] = row\n failures = []\n # OPS-6: Support Found=Y, Conflict=Y, notes with 7100 and 8100\n if 'OPS-6' not in rows:\n failures.append('OPS-6: not found')\n else:\n row = rows['OPS-6']\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-6: Support Found={sf} (expected Y)')\n if conflict not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-6: Conflict={conflict} (expected Y)')\n has_7100 = '7100' in notes or 'capital' in notes\n has_8100 = '8100' in notes or 'it' in notes or 'equipment' in notes\n if not has_7100:\n failures.append(f'OPS-6: Resolution Notes missing GL code 7100/Capital-Equipment: \"{notes}\"')\n if not has_8100:\n failures.append(f'OPS-6: Resolution Notes missing GL code 8100/IT-Equipment: \"{notes}\"')\n # OPS-10: Support Found=Y, Conflict=Y, notes with 6200-Office-Supplies and 6200-Office-Exp\n if 'OPS-10' not in rows:\n failures.append('OPS-10: not found')\n else:\n row = rows['OPS-10']\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-10: Support Found={sf} (expected Y)')\n if conflict not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-10: Conflict={conflict} (expected Y)')\n has_6200 = '6200' in notes\n has_office = 'office' in notes\n if not has_6200:\n failures.append(f'OPS-10: Resolution Notes missing GL code 6200: \"{notes}\"')\n if not has_office:\n failures.append(f'OPS-10: Resolution Notes missing Office reference: \"{notes}\"')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'OPS-6 and OPS-10 both show Support Found=Y, Conflict=Y, and Resolution Notes documenting both GL codes'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
"criterion_type": "expected_output"
},
{
"id": "a5a058cc-1ac7-466b-8799-e7bc4162bba9",
"sort_order": 6,
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-9 must show Support Found=Y, Conflict=N, Status=EXCEPTION, and Resolution Notes indicating missing Slack approval.",
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h.startswith('support'):\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [status_col, support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns. Headers: {headers}'}\n susp_008_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id == 'OPS-9':\n susp_008_row = row\n break\n if susp_008_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'OPS-9 not found in workbook'}\n row = susp_008_row\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n status = str(row[status_col]).strip().upper() if row[status_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n failures = []\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'Support Found={sf} (expected Y)')\n if conflict not in ('N', 'NO', 'FALSE', '0'):\n failures.append(f'Conflict={conflict} (expected N)')\n if 'EXCEPTION' not in status:\n failures.append(f'Status={status} (expected EXCEPTION)')\n if not any(kw in notes for kw in ['slack', 'approval', 'missing']):\n failures.append(f'Resolution Notes does not indicate missing Slack approval: \"{notes}\"')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'OPS-9 failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'OPS-9 shows Support Found=Y, Conflict=N, Status=EXCEPTION, and Resolution Notes indicating missing Slack approval'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
"criterion_type": "expected_output"
},
{
"id": "5f6dad05-f5cf-4e49-9cb1-0f36e9672343",
"sort_order": 7,
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-2, OPS-3, OPS-7, and OPS-8 must each have at least one comment in that list containing the transaction ID, the target GL code, and the phrase \"FIN-100 Cleared & Posted\" (case-insensitive).ve).",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Look for jira_state.json in external_services_path first, then workspace\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in workspace or external_services_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n target_ids = ['OPS-2', 'OPS-3', 'OPS-7', 'OPS-8']\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n # So we expect jira_data to have a top-level key \"comments\" which is a dict keyed by issue ID\n # e.g. { \"comments\": { \"OPS-2\": [...], \"OPS-3\": [...], ... }, ... }\n \n # Try to find the comments object\n comments_obj = None\n if isinstance(jira_data, dict):\n # Direct top-level \"comments\" key\n if 'comments' in jira_data:\n comments_obj = jira_data['comments']\n # Maybe nested under another key\n if comments_obj is None:\n for k, v in jira_data.items():\n if isinstance(v, dict) and any(tid in v or tid.lower() in str(v).lower() for tid in target_ids):\n # Check if this looks like a comments-by-issue-id dict\n if any(isinstance(vv, list) for vv in v.values()):\n comments_obj = v\n break\n \n # Fallback: search the entire structure for comment lists\n if comments_obj is None:\n # Try to find any dict that has issue IDs as keys mapping to lists\n def find_comments_dict(obj, depth=0):\n if depth > 5:\n return None\n if isinstance(obj, dict):\n # Check if this dict itself has target IDs as keys mapping to lists\n matching = sum(1 for tid in target_ids if tid in obj and isinstance(obj.get(tid), list))\n if matching >= 1:\n return obj\n for k, v in obj.items():\n result = find_comments_dict(v, depth + 1)\n if result is not None:\n return result\n return None\n comments_obj = find_comments_dict(jira_data)\n\n if comments_obj is None or not isinstance(comments_obj, dict):\n return {'pass': False, 'score': 0.0, 'feedback': f'No top-level comments object found in jira_state.json. Top-level keys: {list(jira_data.keys()) if isinstance(jira_data, dict) else \"not a dict\"}'}\n\n failures = []\n successes = []\n\n for tid in target_ids:\n tid_lower = tid.lower()\n\n # Find the comment list for this issue ID (case-insensitive key match)\n comment_list = None\n for key, val in comments_obj.items():\n if key.lower() == tid_lower or key.upper() == tid.upper():\n comment_list = val\n break\n \n if comment_list is None:\n # Also try looking inside issues structure for comments\n failures.append(f'{tid}: no comment list found keyed by this issue ID in the comments object')\n continue\n\n if not isinstance(comment_list, list):\n comment_list = [comment_list]\n\n # Check each comment for required content:\n # 1. transaction ID (the SUSP-XXX)\n # 2. target GL code (some numeric code or GL reference)\n # 3. phrase \"FIN-100 Cleared & Posted\" (case-insensitive)\n found_valid_comment = False\n best_comment_info = ''\n for comment in comment_list:\n comment_text = _get_comment_text(comment)\n comment_lower = comment_text.lower()\n\n # Check for transaction ID\n has_tid = tid_lower in comment_lower\n\n # Check for \"FIN-100 Cleared & Posted\" (case-insensitive, flexible on & vs and)\n has_fin100_cleared_posted = bool(\n re.search(r'fin[\\-\\s]?100\\s+cleared\\s*[&and]+\\s*posted', comment_lower)\n )\n # More lenient: just check for fin-100 and cleared and posted separately\n if not has_fin100_cleared_posted:\n has_fin100 = bool(re.search(r'fin[\\-\\s]?100', comment_lower))\n has_cleared = 'cleared' in comment_lower\n has_posted = 'posted' in comment_lower\n has_fin100_cleared_posted = has_fin100 and has_cleared and haLine truncated
"criterion_type": "expected_output"
},
{
"id": "2e4c68b9-6fc5-4ca9-9c09-bc2f3cb0ddf4",
"sort_order": 8,
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-6 must have at least one comment containing both GL codes \"7100\" and \"8100\". OPS-10 must have at least one comment containing both \"6200\" and \"office\". Comments are stored in Atlassian doc format: extract text by walking body → content[] → content[] → text fields.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef extract_adf_text(node):\n \"\"\"Recursively extract text from Atlassian Document Format nodes.\"\"\"\n texts = []\n if isinstance(node, dict):\n if 'text' in node:\n texts.append(str(node['text']))\n for key in ('content', 'body'):\n if key in node:\n val = node[key]\n if isinstance(val, list):\n for item in val:\n texts.extend(extract_adf_text(item))\n elif isinstance(val, dict):\n texts.extend(extract_adf_text(val))\n elif isinstance(node, list):\n for item in node:\n texts.extend(extract_adf_text(item))\n return texts\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / 'jira_state.json')\n candidates.append(Path(workspace_path) / 'jira_state.json')\n for c in candidates:\n if c.exists():\n jira_path = c\n break\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n # So we look for jira_data[\"comments\"] as a dict keyed by issue ID like \"OPS-6\"\n comments_obj = None\n if isinstance(jira_data, dict):\n # Try direct top-level \"comments\" key\n if 'comments' in jira_data and isinstance(jira_data['comments'], dict):\n comments_obj = jira_data['comments']\n\n # Fallback: search the entire structure for a dict that has keys matching SUSP-xxx\n if comments_obj is None and isinstance(jira_data, dict):\n for key, val in jira_data.items():\n if isinstance(val, dict):\n # Check if any sub-key looks like an issue ID\n for subkey in val:\n if isinstance(subkey, str) and subkey.upper().startswith('SUSP-'):\n comments_obj = val\n break\n if comments_obj:\n break\n\n checks = {\n 'OPS-6': ['7100', '8100'],\n 'OPS-10': ['6200', 'office']\n }\n failures = []\n details = []\n\n for issue_id, keywords in checks.items():\n # Try to find comments for this issue\n comment_list = None\n if comments_obj is not None:\n # Try exact key and case-insensitive\n for k, v in comments_obj.items():\n if k.upper() == issue_id.upper():\n comment_list = v if isinstance(v, list) else [v]\n break\n\n # Fallback: look through issues list for embedded comments\n if comment_list is None and isinstance(jira_data, dict):\n issues = []\n if 'issues' in jira_data and isinstance(jira_data['issues'], list):\n issues = jira_data['issues']\n elif 'subtasks' in jira_data and isinstance(jira_data['subtasks'], list):\n issues = jira_data['subtasks']\n for issue in issues:\n if isinstance(issue, dict):\n issue_str = json.dumps(issue).lower()\n if issue_id.lower() in issue_str:\n if 'comments' in issue:\n comment_list = issue['comments'] if isinstance(issue['comments'], list) else [issue['comments']]\n elif 'comment' in issue:\n c = issue['comment']\n comment_list = c if isinstance(c, list) else [c]\n break\n\n if comment_list is None or len(comment_list) == 0:\n failures.append(f'{issue_id}: no comments found for this issue')\n continue\n\n # For each comment, extract text using ADF walking and also fallback to full JSON dump\n found_comment_with_all_keywords = False\n for comment in comment_list:\n # Extract text via ADF\n adf_texts = extract_adf_text(comment)\n combined_adf = ' '.join(adf_texts).lower()\n\n # Also get full JSON dump as fallback\n full_dump = json.dumps(comment).lower()\n\n # Also check plain string body\n body_text = ''\n if isinstance(comment, dict):\n body_val = comment.get('body', '')\n if isinstance(body_val, str):\n body_text = body_val.lower()\n\n # Combine all text sources\n all_texLine truncated
"criterion_type": "expected_output"
},
{
"id": "b607ef6c-73ef-4e16-8c21-366d6d96d4df",
"sort_order": 9,
"rubric_text": "In jira_state.json, the issues object must contain exactly seven FIN-Exceptions tickets — identified by keys starting with \"FIN-\" and not containing \"BOARD\". One ticket must exist for each of: OPS-4 (missing Slack approval), OPS-5 (missing PDF), OPS-6 (GL conflict), OPS-9 (missing Slack approval), OPS-10 (GL conflict), OPS-11 (missing Slack approval), and OPS-12 (self-approval). Each ticket must reference its corresponding SUSP ID somewhere in its content.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n # The rubric says: \"the issues object must contain exactly seven FIN-Exceptions tickets\n # identified by keys starting with FIN- and not containing BOARD\"\n # So we look for jira_data[\"issues\"] as a dict keyed by ticket IDs.\n issues = None\n if isinstance(jira_data, dict):\n # Try to find an \"issues\" key\n if 'issues' in jira_data:\n issues = jira_data['issues']\n else:\n # Maybe the top-level dict itself is the issues dict\n issues = jira_data\n\n if issues is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find issues object in jira_state.json'}\n\n # Collect FIN- tickets that don't contain BOARD in their key\n fin_exception_tickets = {}\n\n if isinstance(issues, dict):\n for key, val in issues.items():\n key_upper = key.upper()\n if key_upper.startswith('FIN-') and 'BOARD' not in key_upper:\n fin_exception_tickets[key] = val\n elif isinstance(issues, list):\n # If issues is a list, look for key/id fields\n for item in issues:\n if isinstance(item, dict):\n key = str(item.get('key', '') or item.get('id', '') or '')\n key_upper = key.upper()\n if key_upper.startswith('FIN-') and 'BOARD' not in key_upper:\n fin_exception_tickets[key] = item\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'issues object is of unexpected type: {type(issues)}'}\n\n # Check count\n num_tickets = len(fin_exception_tickets)\n failures = []\n if num_tickets != 7:\n failures.append(f'Expected exactly 7 FIN-Exception tickets (keys starting with FIN- and not containing BOARD), found {num_tickets}. Keys found: {list(fin_exception_tickets.keys())}')\n\n # Required SUSP IDs\n required_ids = ['OPS-4', 'OPS-5', 'OPS-6', 'OPS-9', 'OPS-10', 'OPS-11', 'OPS-12']\n\n # For each required SUSP ID, check that at least one FIN-Exception ticket references it\n found_ids = set()\n for rid in required_ids:\n for key, val in fin_exception_tickets.items():\n # Serialize the entire ticket value to search for the SUSP ID\n ticket_str = json.dumps(val) if not isinstance(val, str) else val\n # Also check the key itself\n full_str = key + ' ' + ticket_str\n if rid in full_str:\n found_ids.add(rid)\n break\n\n missing_ids = [rid for rid in required_ids if rid not in found_ids]\n if missing_ids:\n failures.append(f'Missing FIN-Exception tickets referencing these SUSP IDs: {missing_ids}')\n\n if failures:\n score = len(found_ids) / len(required_ids) if not (num_tickets != 7 and len(found_ids) == 0) else 0.0\n return {'pass': False, 'score': score, 'feedback': 'Failures: ' + '; '.join(failures)}\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found exactly 7 FIN-Exception tickets with keys {list(fin_exception_tickets.keys())}, covering all required SUSP IDs: {required_ids}'\n }\n",
"criterion_type": "expected_output"
},
{
"id": "e720abd4-86ab-4a80-80b3-74074d7a2b1e",
"sort_order": 10,
"rubric_text": "In jira_state.json, all FIN-Exceptions tickets (for OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, OPS-12) must be assigned to jennifer.walsh.",
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef _flatten_issues(jira_data):\n \"\"\"Extract all individual issue dicts from the jira_state structure.\"\"\"\n all_issues = []\n if not isinstance(jira_data, dict):\n if isinstance(jira_data, list):\n return [i for i in jira_data if isinstance(i, dict)]\n return []\n\n for key, val in jira_data.items():\n if key == 'comments':\n continue\n if isinstance(val, list):\n for item in val:\n if isinstance(item, dict):\n all_issues.append(item)\n elif isinstance(val, dict):\n if any(k in val for k in ('key', 'fields', 'id', 'assignee', 'summary')):\n all_issues.append(val)\n else:\n for sub_key, sub_val in val.items():\n if isinstance(sub_val, dict):\n all_issues.append(sub_val)\n return all_issues\n\n\ndef _check_assignee_for_jennifer(ticket):\n \"\"\"Check ONLY the assignee field of a ticket for jennifer.walsh.\"\"\"\n jennifer_variants = ['jennifer.walsh', 'jennifer walsh', 'jennifer_walsh', 'jwalsh', 'j.walsh']\n\n if not isinstance(ticket, dict):\n return False\n\n containers = [ticket]\n if isinstance(ticket.get('fields'), dict):\n containers.append(ticket['fields'])\n\n for container in containers:\n for field_name in ['assignee', 'assigned_to', 'owner']:\n val = container.get(field_name)\n if val is None:\n continue\n if isinstance(val, str):\n if any(v in val.lower() for v in jennifer_variants):\n return True\n elif isinstance(val, dict):\n for subfield in ['accountId', 'displayName', 'emailAddress', 'name', 'key']:\n subval = val.get(subfield, '')\n if subval and isinstance(subval, str):\n if any(v in subval.lower() for v in jennifer_variants):\n return True\n return False\n\n\ndef _get_assignee_display(ticket):\n \"\"\"Extract a human-readable assignee string for error messages.\"\"\"\n if not isinstance(ticket, dict):\n return 'unknown'\n for container in [ticket.get('fields', {}), ticket]:\n if not isinstance(container, dict):\n continue\n for field_name in ['assignee', 'assigned_to', 'owner']:\n val = container.get(field_name)\n if val is None:\n continue\n if isinstance(val, dict):\n return val.get('displayName', val.get('accountId', str(val)))\n if isinstance(val, str) and val:\n return val\n return 'unknown'\n\n\ndef _get_issue_key(issue):\n \"\"\"Return the issue key (e.g. 'FIN-3') if present.\"\"\"\n if not isinstance(issue, dict):\n return ''\n key = issue.get('key')\n if isinstance(key, str):\n return key\n fields = issue.get('fields')\n if isinstance(fields, dict):\n k = fields.get('key')\n if isinstance(k, str):\n return k\n return ''\n\n\ndef _get_project_key(issue):\n \"\"\"Return the project key (e.g. 'FIN') if present.\"\"\"\n if not isinstance(issue, dict):\n return ''\n fields = issue.get('fields') if isinstance(issue.get('fields'), dict) else {}\n proj = fields.get('project') if isinstance(fields, dict) else None\n if isinstance(proj, dict):\n pk = proj.get('key')\n if isinstance(pk, str):\n return pk\n return ''\n\n\ndef _summary_and_description_text(issue):\n \"\"\"Return summary + description text for matching SUSP-IDs against, lowercased.\"\"\"\n if not isinstance(issue, dict):\n return ''\n fields = issue.get('fields') if isinstance(issue.get('fields'), dict) else issue\n parts = []\n summary = fields.get('summary')\n if isinstance(summary, str):\n parts.append(summary)\n desc = fields.get('description')\n if isinstance(desc, str):\n parts.append(desc)\n elif isinstance(desc, dict):\n # Atlassian Document Format — just dump it; SUSP-XXX will appear as a substring\n parts.append(json.dumps(desc))\n return ' '.join(parts)\n\n\ndef verify(workspace_path, external_services_path=None):\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n\n all_issues = _flatten_issues(jira_data)\Line truncated
"criterion_type": "expected_output"
},
{
"id": "eb714737-6b44-4948-b983-7d1f811133c5",
"sort_order": 11,
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-4, OPS-5, OPS-9, OPS-11, and OPS-12 must each have at least one comment in that list containing an escalation reason. Comments are stored in Atlassian doc format: extract text by walking body → content[] → content[] → text fields. Acceptable keywords indicating an escalation reason include: escalat, exception, missing, conflict, approval, pdf, slack, self, manager.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef _extract_adf_text(node):\n \"\"\"Recursively extract text from Atlassian Document Format nodes.\"\"\"\n texts = []\n if isinstance(node, dict):\n if 'text' in node:\n texts.append(str(node['text']))\n for key in ('body', 'content', 'children'):\n child = node.get(key)\n if isinstance(child, list):\n for item in child:\n texts.extend(_extract_adf_text(item))\n elif isinstance(child, dict):\n texts.extend(_extract_adf_text(child))\n elif isinstance(node, list):\n for item in node:\n texts.extend(_extract_adf_text(item))\n return texts\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n if not isinstance(jira_data, dict):\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json top-level is not a dict'}\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n comments_obj = jira_data.get('comments', {})\n if not isinstance(comments_obj, dict):\n # Fallback: maybe the entire jira_data acts as a container; try to find comments elsewhere\n comments_obj = {}\n\n # Also collect all issues from other top-level keys (issues list, etc.) for fallback\n all_issues = []\n for key, val in jira_data.items():\n if key == 'comments':\n continue\n if isinstance(val, list):\n all_issues.extend(val)\n elif isinstance(val, dict):\n all_issues.append(val)\n\n required_ids = ['OPS-4', 'OPS-5', 'OPS-9', 'OPS-11', 'OPS-12']\n escalation_keywords = ['escalat', 'exception', 'missing', 'conflict', 'approval', 'pdf', 'slack', 'self', 'manager']\n\n failures = []\n successes = []\n\n for tid in required_ids:\n combined_text = ''\n\n # Primary path: top-level comments object keyed by issue ID\n comment_list = comments_obj.get(tid, [])\n if not isinstance(comment_list, list):\n comment_list = [comment_list] if comment_list else []\n\n for comment in comment_list:\n # Extract text via ADF walking\n adf_texts = _extract_adf_text(comment)\n combined_text += ' '.join(adf_texts) + ' '\n # Also grab any plain string fields\n if isinstance(comment, dict):\n for field in ('body', 'text', 'content'):\n v = comment.get(field)\n if isinstance(v, str):\n combined_text += v + ' '\n elif isinstance(comment, str):\n combined_text += comment + ' '\n\n # Fallback: look through issues for matching issue and its comments\n if not combined_text.strip():\n for issue in all_issues:\n if not isinstance(issue, dict):\n continue\n issue_str = json.dumps(issue)\n if tid in issue_str:\n # Try to get comments from the issue object\n issue_comments = issue.get('comments', issue.get('comment', []))\n if not isinstance(issue_comments, list):\n issue_comments = [issue_comments] if issue_comments else []\n for comment in issue_comments:\n adf_texts = _extract_adf_text(comment)\n combined_text += ' '.join(adf_texts) + ' '\n if isinstance(comment, dict):\n for field in ('body', 'text', 'content'):\n v = comment.get(field)\n if isinstance(v, str):\n combined_text += v + ' '\n elif isinstance(comment, str):\n combined_text += comment + ' '\n # Also check description/notes as last resort\n if not combined_text.strip():\n desc = str(issue.get('description', '') or '') + ' ' + str(issue.get('notes', '') or '')\n combined_text += desc\n break\n\n Line truncated
"criterion_type": "expected_output"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1,4 @@
Hey, I need you to handle the March intercompany suspense reconciliation. We're already a few days past the month close deadline and Marcus is asking for an update on where we stand with the suspense items.
The suspense workbook and SOP are in the workspace. Follow the SOP procedures for intercompany suspense reconciliation.
Assume it's April 4th, 2026. Save your completed reconciliation workbook to Suspense_Reconciliation_March2026.xlsx. Add these specific tie-out columns to the Excel file: Support Found, Support Location, Conflict, Status, and Resolution Notes.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py