DerekSurge
87 lines
68 KiB
JSON
87 lines
68 KiB
JSON
[
|
|
{
|
|
"id": "cd482ae3-5a98-473d-944b-90077a455e30",
|
|
"sort_order": 0,
|
|
"rubric_text": "File `Suspense_Reconciliation_March2026.xlsx` must exist in the workspace and contain columns: Support Found, Support Location, Conflict, Status, and Resolution Notes.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip() if cell.value else '' for cell in ws[1]]\n headers_lower = [h.lower() for h in headers]\n required_cols = ['support found', 'support location', 'conflict', 'status', 'resolution notes']\n missing = [col for col in required_cols if col not in headers_lower]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns: {missing}. Found columns: {headers}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'File exists and contains all required columns: {required_cols}'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading file: {e}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "294d7b7f-70c3-46e0-993b-71b9f0cecf2c",
|
|
"sort_order": 1,
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, cleared items OPS-2 ($3,250), OPS-3 ($7,800), OPS-7 ($2,100), and OPS-8 ($15,000) must each show Status=FIN-100, Support Found=Y, Conflict=N, and non-empty Resolution Notes.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n def col_idx(name):\n for i, h in enumerate(headers):\n if name.lower() in h:\n return i\n return None\n id_col = col_idx('susp') if col_idx('susp') is not None else col_idx('id') if col_idx('id') is not None else 0\n # Try to find transaction ID column\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or (i > 0 and 'support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h == 'support found':\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [status_col, support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find required columns. Headers: {headers}'}\n target_ids = {'OPS-2', 'OPS-3', 'OPS-7', 'OPS-8'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n status = str(row[status_col]).strip() if row[status_col] else ''\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip() if row[notes_col] else ''\n if 'FIN-100' not in status.upper() and status.upper() != 'FIN-100':\n failures.append(f'{tid}: Status={status} (expected FIN-100)')\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'{tid}: Support Found={sf} (expected Y)')\n if conflict not in ('N', 'NO', 'FALSE', '0'):\n failures.append(f'{tid}: Conflict={conflict} (expected N)')\n if not notes or notes.lower() == 'none':\n failures.append(f'{tid}: Resolution Notes is empty')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': f'All cleared items (OPS-2, OPS-3, OPS-7, OPS-8) show Status=FIN-100, Support Found=Y, Conflict=N, and non-empty Resolution Notes'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "731b9a6e-3474-439c-95c7-5855d1a54007",
|
|
"sort_order": 2,
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-3 ($7,800) and OPS-8 ($15,000) Resolution Notes must reference a Slack approval message.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if notes_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Resolution Notes column. Headers: {headers}'}\n target_ids = {'OPS-3', 'OPS-8'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n slack_keywords = ['slack', 'approval', 'approved', '#finance', 'channel', 'message']\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if not any(kw in notes for kw in slack_keywords):\n failures.append(f'{tid}: Resolution Notes does not reference Slack approval. Notes: \"{notes}\"')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'OPS-3 and OPS-8 Resolution Notes reference Slack approval message'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "753ac24a-0b95-46ad-9b11-a3a2dd24d630",
|
|
"sort_order": 3,
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, exception items OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, and OPS-12 must each show Status=EXCEPTION and must NOT show Status=FIN-100.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n if status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Status column. Headers: {headers}'}\n target_ids = {'OPS-4', 'OPS-5', 'OPS-6', 'OPS-9', 'OPS-10', 'OPS-11', 'OPS-12'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n status = str(row[status_col]).strip().upper() if row[status_col] else ''\n if 'EXCEPTION' not in status:\n failures.append(f'{tid}: Status={status} (expected EXCEPTION)')\n if 'FIN-100' in status:\n failures.append(f'{tid}: Status={status} must NOT be FIN-100')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'All exception items (OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, OPS-12) show Status=EXCEPTION'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "e23f7481-1554-4d6d-9055-770e06854a17",
|
|
"sort_order": 4,
|
|
"rubric_text": "In file Suspense_Reconciliation_March2026.xlsx, OPS-4 must show Support Found=Y, Conflict=N, and Resolution Notes indicating missing Slack approval. OPS-5 must show Support Found=N, Conflict=N, and Resolution Notes indicating missing PDF. OPS-11 must show Support Found=Y, Conflict=N, and Resolution Notes indicating no manager approval found in #finance-approvals. OPS-12 must show Support Found=Y, Conflict=N, and Resolution Notes indicating approval must be confirmed by a manager, not a junior analyst.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file - be flexible\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n candidates = list(Path(workspace_path).glob('*Suspense*Reconciliation*March*2026*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*reconciliation*march*2026*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*Suspense*March*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*march*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*Suspense*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*.xlsx'))\n if candidates:\n xlsx_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = None\n for sheet_name in wb.sheetnames:\n candidate_ws = wb[sheet_name]\n first_row = list(candidate_ws.iter_rows(min_row=1, max_row=1, values_only=True))\n if not first_row:\n continue\n candidate_headers = [str(cell).strip().lower() if cell else '' for cell in first_row[0]]\n has_support = any('support' in h for h in candidate_headers)\n has_conflict = any('conflict' in h for h in candidate_headers)\n has_notes = any('resolution' in h or 'notes' in h for h in candidate_headers)\n if has_support and has_conflict and has_notes:\n ws = candidate_ws\n break\n if ws is None:\n ws = wb.active\n\n first_row = list(ws.iter_rows(min_row=1, max_row=1, values_only=True))\n if not first_row or not first_row[0]:\n return {'pass': False, 'score': 0.0, 'feedback': 'No headers found in first row'}\n headers = [str(cell).strip().lower() if cell else '' for cell in first_row[0]]\n\n id_col = None\n for i, h in enumerate(headers):\n if 'susp' in h or h == 'id' or 'item id' in h or 'item_id' in h:\n id_col = i\n break\n if id_col is None:\n for i, h in enumerate(headers):\n if 'item' in h or 'transaction' in h or 'entry' in h or 'ref' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if 'support' in h and 'location' not in h and 'loc' not in h:\n support_found_col = i\n break\n\n support_location_col = None\n for i, h in enumerate(headers):\n if 'support location' in h or ('support' in h and 'location' in h) or ('support' in h and 'loc' in h):\n support_location_col = i\n break\n\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n\n status_col = None\n for i, h in enumerate(headers):\n if h == 'status' or 'status' in h:\n status_col = i\n break\n\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h and 'note' in h:\n notes_col = i\n break\n if notes_col is None:\n for i, h in enumerate(headers):\n if 'resolution' in h:\n notes_col = i\n break\n if notes_col is None:\n for i, h in enumerate(headers):\n if 'notes' in h or 'note' in h:\n notes_col = i\n break\n if notes_col is None:\n for i, h in enumerate(headers):\n if 'comment' in h or 'remark' in h:\n notes_col = i\n break\n\n missing_cols = []\n if support_found_col is None:\n missing_cols.append('Support Found')\n if conflict_col is None:\n missing_cols.append('Conflict')\n if notes_col is None:\n missing_cols.append('Resolution Notes')\n if missing_cols:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns: {missing_cols}. Found headers: {headers}'}\n\n rows = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n if row is None:\n continue\n row_id = ''\n if id_col < len(row) and row[id_col]:\n cell_str = str(row[id_col]).strip().upper()\n if 'SUSP' in cell_str:\n import re\n match = re.search(r'SUSP-?\\d+', cell_str)\n if match:\n row_id = match.group(0)\n row_id = row_id.replace('SUSP', 'SUSP-').replace('SUSP--', 'SUSP-')\n if not row_id:\n for cell_val in row:\n if cell_val:\n cell_str = str(cell_val).strip().upper()\n if 'SUSP' in cell_str:\n import re\n match = re.search(r'SUSP-?\\d+', cell_str)\n if match:\n row_id = match.group(0)\n row_id = row_id.replace('SUSP', 'SUSP-').replace('SUSP--', 'SUSP-')\n break\n if row_id in ('OPS-4', 'OPS-5', 'OPS-11', 'OPS-12'):\n rows[row_id] = row\n\n failures = []\n details = []\n\n def check_yn(value, expected_yes):\n \"\"\"Strict check: only Y/Yes count as yes; only N/No count as no.\"\"\"\n val = str(value).strip().upper() if value is not None else ''\n if expected_yes:\n return val in ('Y', 'YES')\n else:\n return val in ('N', 'NO')\n\n def safe_get(row, col):\n if col is not None and col < len(row):\n return row[col]\n return None\n\n # OPS-4\n if 'OPS-4' not in rows:\n failures.append('OPS-4: row not found in spreadsheet')\n else:\n row = rows['OPS-4']\n sf_val = safe_get(row, support_found_col)\n conflict_val = safe_get(row, conflict_col)\n notes_val = safe_get(row, notes_col)\n sf = str(sf_val).strip().upper() if sf_val else ''\n conflict = str(conflict_val).strip().upper() if conflict_val else ''\n notes = str(notes_val).strip().lower() if notes_val else ''\n if not check_yn(sf_val, True):\n failures.append(f'OPS-4: Support Found=\"{sf}\" (expected Y)')\n if not check_yn(conflict_val, False):\n failures.append(f'OPS-4: Conflict=\"{conflict}\" (expected N)')\n slack_keywords = ['slack', '#finance', 'channel', 'message']\n approval_keywords = ['approval', 'approved', 'authorize', 'authoriz', 'confirm']\n missing_keywords = ['missing', 'no ', 'not ', 'absent', 'lack', 'without', 'pending', 'needed', 'unable', 'could not', \"couldn't\", 'unavailable']\n has_slack = any(kw in notes for kw in slack_keywords)\n has_approval = any(kw in notes for kw in approval_keywords)\n has_missing = any(kw in notes for kw in missing_keywords)\n if not (has_slack or (has_approval and has_missing)):\n failures.append(f'OPS-4: Resolution Notes does not indicate missing Slack approval: \"{notes[:120]}\"')\n else:\n details.append(f'OPS-4: OK (SF={sf}, Conflict={conflict}, Notes=\"{notes[:80]}\")')\n\n # OPS-5\n if 'OPS-5' not in rows:\n failures.append('OPS-5: row not found in spreadsheet')\n else:\n row = rows['OPS-5']\n sf_val = safe_get(row, support_found_col)\n conflict_val = safe_get(row, conflict_col)\n notes_val = safe_get(row, notes_col)\n sf = str(sf_val).strip().upper() if sf_val else ''\n conflict = str(conflict_val).strip().upper() if conflict_val else ''\n notes = str(notes_val).strip().lower() if notes_val else ''\n if not check_yn(sf_val, False):\n failures.append(f'OPS-5: Support Found=\"{sf}\" (expected N)')\n if not check_yn(conflict_val, False):\n failures.append(f'OPS-5: Conflict=\"{conflict}\" (expected N)')\n pdf_keywords = ['pdf', 'document', 'file', 'attachment', 'invoice']\n missing_kw = ['missing', 'no ', 'not ', 'absent', 'lack', 'without', 'unavailable', 'unable', 'could not', \"couldn't\", 'not found', 'pending']\n support_kw = ['support', 'documentation', 'evidence', 'backup']\n has_pdf = any(kw in notes for kw in pdf_keywords)\n has_missing2 = any(kw in notes for kw in missing_kw)\n has_support = any(kw in notes for kw in support_kw)\n if not ((has_pdf and has_missing2) or (has_missing2 and has_support) or has_pdf):\n failures.append(f'OPS-5: Resolution Notes does not indicate missing PDF: \"{notes[:120]}\"')\n else:\n details.append(f'OPS-5: OK (SF={sf}, Conflict={conflict}, Notes=\"{notes[:80]}\")')\n\n # OPS-11\n if 'OPS-11' not in rows:\n failures.append('OPS-11: row not found in spreadsheet')\n else:\n row = rows['OPS-11']\n sf_val = safe_get(row, support_found_col)\n conflict_val = safe_get(row, conflict_col)\n notes_val = safe_get(row, notes_col)\n sf = str(sf_val).strip().upper() if sf_val else ''\n conflict = str(conflict_val).strip().upper() if conflict_val else ''\n notes = str(notes_val).strip().lower() if notes_val else ''\n if not check_yn(sf_val, True):\n failures.append(f'OPS-11: Support Found=\"{sf}\" (expected Y)')\n if not check_yn(conflict_val, False):\n failures.append(f'OPS-11: Conflict=\"{conflict}\" (expected N)')\n manager_kw = ['manager', 'supervisor', 'management']\n approval_kw = ['approval', 'approved', 'authorize', 'authoriz']\n finance_kw = ['finance', '#finance', 'finance-approval']\n missing_kw2 = ['no ', 'not ', 'missing', 'absent', 'without', 'lack', 'pending', 'unable', 'needed']\n has_manager = any(kw in notes for kw in manager_kw)\n has_approval2 = any(kw in notes for kw in approval_kw)\n has_finance = any(kw in notes for kw in finance_kw)\n has_missing3 = any(kw in notes for kw in missing_kw2)\n if not ((has_manager or has_approval2) and (has_finance or has_missing3)):\n failures.append(f'OPS-11: Resolution Notes does not indicate no manager approval in #finance-approvals: \"{notes[:120]}\"')\n else:\n details.append(f'OPS-11: OK (SF={sf}, Conflict={conflict}, Notes=\"{notes[:80]}\")')\n\n # OPS-12\n if 'OPS-12' not in rows:\n failures.append('OPS-12: row not found in spreadsheet')\n else:\n row = rows['OPS-12']\n sf_val = safe_get(row, support_found_col)\n conflict_val = safe_get(row, conflict_col)\n notes_val = safe_get(row, notes_col)\n sf = str(sf_val).strip().upper() if sf_val else ''\n conflict = str(conflict_val).strip().upper() if conflict_val else ''\n notes = str(notes_val).strip().lower() if notes_val else ''\n if not check_yn(sf_val, True):\n failures.append(f'OPS-12: Support Found=\"{sf}\" (expected Y)')\n if not check_yn(conflict_val, False):\n failures.append(f'OPS-12: Conflict=\"{conflict}\" (expected N)')\n manager_kw2 = ['manager', 'supervisor', 'senior', 'management']\n junior_kw = ['junior', 'analyst', 'jr', 'associate']\n approval_kw2 = ['approval', 'approved', 'confirm', 'authorize', 'valid', 'invalid', 'self']\n sop_kw = ['sop', 'policy', 'procedure', 'requirement', 'authority', 'unauthorized', 'insufficient']\n has_manager2 = any(kw in notes for kw in manager_kw2)\n has_junior = any(kw in notes for kw in junior_kw)\n has_approval3 = any(kw in notes for kw in approval_kw2)\n has_sop = any(kw in notes for kw in sop_kw)\n negative_kw = ['missing', 'insufficient', 'invalid', 'not a manager', 'not manager', 'escalat', 'exception', 'unauthorized', 'does not', 'cannot', 'should not', 'must be', 'need', 'require', 'lacking', 'improper', 'flag', 'issue', 'concern', 'problem']\n has_negative = any(kw in notes for kw in negative_kw)\n if not ((has_junior or has_manager2) and (has_approval3 or has_sop) and (has_negative or has_sop)):\n failures.append(f'OPS-12: Resolution Notes does not indicate approval must be by manager not junior analyst: \"{notes[:120]}\"')\n else:\n details.append(f'OPS-12: OK (SF={sf}, Conflict={conflict}, Notes=\"{notes[:80]}\")')\n\n if failures:\n score = max(0.0, 1.0 - (len(failures) / 12.0))\n return {'pass': False, 'score': score, 'feedback': 'Failures: ' + '; '.join(failures)}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'All checks passed. ' + '; '.join(details)}\n except Exception as e:\n import traceback\n return {'pass': False, 'score': 0.0, 'feedback': f'Error processing file: {e}\\n{traceback.format_exc()}'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "8c35192a-a420-4647-9f7b-a056ddad0a04",
|
|
"sort_order": 5,
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-6 must show Support Found=Y, Conflict=Y, and Resolution Notes documenting both GL codes (Invoice: 7100-Capital-Equipment and Excel: 8100-IT-Equipment). OPS-10 must show Support Found=Y, Conflict=Y, and Resolution Notes documenting both GL codes (Invoice: 6200-Office-Supplies and Excel: 6200-Office-Exp).",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h.startswith('support'):\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns. Headers: {headers}'}\n rows = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in ('OPS-6', 'OPS-10'):\n rows[row_id] = row\n failures = []\n # OPS-6: Support Found=Y, Conflict=Y, notes with 7100 and 8100\n if 'OPS-6' not in rows:\n failures.append('OPS-6: not found')\n else:\n row = rows['OPS-6']\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-6: Support Found={sf} (expected Y)')\n if conflict not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-6: Conflict={conflict} (expected Y)')\n has_7100 = '7100' in notes or 'capital' in notes\n has_8100 = '8100' in notes or 'it' in notes or 'equipment' in notes\n if not has_7100:\n failures.append(f'OPS-6: Resolution Notes missing GL code 7100/Capital-Equipment: \"{notes}\"')\n if not has_8100:\n failures.append(f'OPS-6: Resolution Notes missing GL code 8100/IT-Equipment: \"{notes}\"')\n # OPS-10: Support Found=Y, Conflict=Y, notes with 6200-Office-Supplies and 6200-Office-Exp\n if 'OPS-10' not in rows:\n failures.append('OPS-10: not found')\n else:\n row = rows['OPS-10']\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-10: Support Found={sf} (expected Y)')\n if conflict not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-10: Conflict={conflict} (expected Y)')\n has_6200 = '6200' in notes\n has_office = 'office' in notes\n if not has_6200:\n failures.append(f'OPS-10: Resolution Notes missing GL code 6200: \"{notes}\"')\n if not has_office:\n failures.append(f'OPS-10: Resolution Notes missing Office reference: \"{notes}\"')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'OPS-6 and OPS-10 both show Support Found=Y, Conflict=Y, and Resolution Notes documenting both GL codes'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "a5a058cc-1ac7-466b-8799-e7bc4162bba9",
|
|
"sort_order": 6,
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-9 must show Support Found=Y, Conflict=N, Status=EXCEPTION, and Resolution Notes indicating missing Slack approval.",
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h.startswith('support'):\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [status_col, support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns. Headers: {headers}'}\n susp_008_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id == 'OPS-9':\n susp_008_row = row\n break\n if susp_008_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'OPS-9 not found in workbook'}\n row = susp_008_row\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n status = str(row[status_col]).strip().upper() if row[status_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n failures = []\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'Support Found={sf} (expected Y)')\n if conflict not in ('N', 'NO', 'FALSE', '0'):\n failures.append(f'Conflict={conflict} (expected N)')\n if 'EXCEPTION' not in status:\n failures.append(f'Status={status} (expected EXCEPTION)')\n if not any(kw in notes for kw in ['slack', 'approval', 'missing']):\n failures.append(f'Resolution Notes does not indicate missing Slack approval: \"{notes}\"')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'OPS-9 failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'OPS-9 shows Support Found=Y, Conflict=N, Status=EXCEPTION, and Resolution Notes indicating missing Slack approval'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "5f6dad05-f5cf-4e49-9cb1-0f36e9672343",
|
|
"sort_order": 7,
|
|
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-2, OPS-3, OPS-7, and OPS-8 must each have at least one comment in that list containing the transaction ID, the target GL code, and the phrase \"FIN-100 Cleared & Posted\" (case-insensitive).ve).",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Look for jira_state.json in external_services_path first, then workspace\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in workspace or external_services_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n target_ids = ['OPS-2', 'OPS-3', 'OPS-7', 'OPS-8']\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n # So we expect jira_data to have a top-level key \"comments\" which is a dict keyed by issue ID\n # e.g. { \"comments\": { \"OPS-2\": [...], \"OPS-3\": [...], ... }, ... }\n \n # Try to find the comments object\n comments_obj = None\n if isinstance(jira_data, dict):\n # Direct top-level \"comments\" key\n if 'comments' in jira_data:\n comments_obj = jira_data['comments']\n # Maybe nested under another key\n if comments_obj is None:\n for k, v in jira_data.items():\n if isinstance(v, dict) and any(tid in v or tid.lower() in str(v).lower() for tid in target_ids):\n # Check if this looks like a comments-by-issue-id dict\n if any(isinstance(vv, list) for vv in v.values()):\n comments_obj = v\n break\n \n # Fallback: search the entire structure for comment lists\n if comments_obj is None:\n # Try to find any dict that has issue IDs as keys mapping to lists\n def find_comments_dict(obj, depth=0):\n if depth > 5:\n return None\n if isinstance(obj, dict):\n # Check if this dict itself has target IDs as keys mapping to lists\n matching = sum(1 for tid in target_ids if tid in obj and isinstance(obj.get(tid), list))\n if matching >= 1:\n return obj\n for k, v in obj.items():\n result = find_comments_dict(v, depth + 1)\n if result is not None:\n return result\n return None\n comments_obj = find_comments_dict(jira_data)\n\n if comments_obj is None or not isinstance(comments_obj, dict):\n return {'pass': False, 'score': 0.0, 'feedback': f'No top-level comments object found in jira_state.json. Top-level keys: {list(jira_data.keys()) if isinstance(jira_data, dict) else \"not a dict\"}'}\n\n failures = []\n successes = []\n\n for tid in target_ids:\n tid_lower = tid.lower()\n\n # Find the comment list for this issue ID (case-insensitive key match)\n comment_list = None\n for key, val in comments_obj.items():\n if key.lower() == tid_lower or key.upper() == tid.upper():\n comment_list = val\n break\n \n if comment_list is None:\n # Also try looking inside issues structure for comments\n failures.append(f'{tid}: no comment list found keyed by this issue ID in the comments object')\n continue\n\n if not isinstance(comment_list, list):\n comment_list = [comment_list]\n\n # Check each comment for required content:\n # 1. transaction ID (the SUSP-XXX)\n # 2. target GL code (some numeric code or GL reference)\n # 3. phrase \"FIN-100 Cleared & Posted\" (case-insensitive)\n found_valid_comment = False\n best_comment_info = ''\n for comment in comment_list:\n comment_text = _get_comment_text(comment)\n comment_lower = comment_text.lower()\n\n # Check for transaction ID\n has_tid = tid_lower in comment_lower\n\n # Check for \"FIN-100 Cleared & Posted\" (case-insensitive, flexible on & vs and)\n has_fin100_cleared_posted = bool(\n re.search(r'fin[\\-\\s]?100\\s+cleared\\s*[&and]+\\s*posted', comment_lower)\n )\n # More lenient: just check for fin-100 and cleared and posted separately\n if not has_fin100_cleared_posted:\n has_fin100 = bool(re.search(r'fin[\\-\\s]?100', comment_lower))\n has_cleared = 'cleared' in comment_lower\n has_posted = 'posted' in comment_lower\n has_fin100_cleared_posted = has_fin100 and has_cleared and has_posted\n\n # Check for GL code - look for numeric codes (4+ digits) or GL/account references\n has_gl = bool(re.search(r'\\d{4,}', comment_text)) or bool(re.search(r'(gl|account|acct|ledger)', comment_lower))\n\n if has_tid and has_fin100_cleared_posted and has_gl:\n found_valid_comment = True\n break\n \n # Track what we found for feedback\n if has_tid or has_fin100_cleared_posted:\n parts = []\n if has_tid: parts.append('has TID')\n if has_fin100_cleared_posted: parts.append('has FIN-100 Cleared & Posted')\n if has_gl: parts.append('has GL code')\n best_comment_info = f' (best match: {\", \".join(parts)}, text preview: {comment_text[:200]})'\n\n if found_valid_comment:\n successes.append(tid)\n else:\n num_comments = len(comment_list)\n failures.append(f'{tid}: no comment (out of {num_comments}) found containing transaction ID, GL code, and \"FIN-100 Cleared & Posted\"{best_comment_info}')\n\n score = len(successes) / len(target_ids)\n\n if failures:\n return {\n 'pass': False,\n 'score': score,\n 'feedback': f'Passed: {successes}. Failures: ' + '; '.join(failures)\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'All cleared items ({\", \".join(target_ids)}) have Jira comments with transaction ID, GL code, and FIN-100 Cleared & Posted'\n }\n\n\ndef _get_comment_text(comment):\n \"\"\"Extract text content from a comment object.\"\"\"\n if isinstance(comment, str):\n return comment\n if isinstance(comment, dict):\n parts = []\n for key in ('body', 'text', 'content', 'message', 'description', 'note', 'comment'):\n if key in comment:\n val = comment[key]\n if isinstance(val, str):\n parts.append(val)\n elif isinstance(val, list):\n parts.append(json.dumps(val))\n elif isinstance(val, dict):\n parts.append(json.dumps(val))\n if not parts:\n return json.dumps(comment)\n return ' '.join(parts)\n return str(comment)\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "2e4c68b9-6fc5-4ca9-9c09-bc2f3cb0ddf4",
|
|
"sort_order": 8,
|
|
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-6 must have at least one comment containing both GL codes \"7100\" and \"8100\". OPS-10 must have at least one comment containing both \"6200\" and \"office\". Comments are stored in Atlassian doc format: extract text by walking body → content[] → content[] → text fields.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef extract_adf_text(node):\n \"\"\"Recursively extract text from Atlassian Document Format nodes.\"\"\"\n texts = []\n if isinstance(node, dict):\n if 'text' in node:\n texts.append(str(node['text']))\n for key in ('content', 'body'):\n if key in node:\n val = node[key]\n if isinstance(val, list):\n for item in val:\n texts.extend(extract_adf_text(item))\n elif isinstance(val, dict):\n texts.extend(extract_adf_text(val))\n elif isinstance(node, list):\n for item in node:\n texts.extend(extract_adf_text(item))\n return texts\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / 'jira_state.json')\n candidates.append(Path(workspace_path) / 'jira_state.json')\n for c in candidates:\n if c.exists():\n jira_path = c\n break\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n # So we look for jira_data[\"comments\"] as a dict keyed by issue ID like \"OPS-6\"\n comments_obj = None\n if isinstance(jira_data, dict):\n # Try direct top-level \"comments\" key\n if 'comments' in jira_data and isinstance(jira_data['comments'], dict):\n comments_obj = jira_data['comments']\n\n # Fallback: search the entire structure for a dict that has keys matching SUSP-xxx\n if comments_obj is None and isinstance(jira_data, dict):\n for key, val in jira_data.items():\n if isinstance(val, dict):\n # Check if any sub-key looks like an issue ID\n for subkey in val:\n if isinstance(subkey, str) and subkey.upper().startswith('SUSP-'):\n comments_obj = val\n break\n if comments_obj:\n break\n\n checks = {\n 'OPS-6': ['7100', '8100'],\n 'OPS-10': ['6200', 'office']\n }\n failures = []\n details = []\n\n for issue_id, keywords in checks.items():\n # Try to find comments for this issue\n comment_list = None\n if comments_obj is not None:\n # Try exact key and case-insensitive\n for k, v in comments_obj.items():\n if k.upper() == issue_id.upper():\n comment_list = v if isinstance(v, list) else [v]\n break\n\n # Fallback: look through issues list for embedded comments\n if comment_list is None and isinstance(jira_data, dict):\n issues = []\n if 'issues' in jira_data and isinstance(jira_data['issues'], list):\n issues = jira_data['issues']\n elif 'subtasks' in jira_data and isinstance(jira_data['subtasks'], list):\n issues = jira_data['subtasks']\n for issue in issues:\n if isinstance(issue, dict):\n issue_str = json.dumps(issue).lower()\n if issue_id.lower() in issue_str:\n if 'comments' in issue:\n comment_list = issue['comments'] if isinstance(issue['comments'], list) else [issue['comments']]\n elif 'comment' in issue:\n c = issue['comment']\n comment_list = c if isinstance(c, list) else [c]\n break\n\n if comment_list is None or len(comment_list) == 0:\n failures.append(f'{issue_id}: no comments found for this issue')\n continue\n\n # For each comment, extract text using ADF walking and also fallback to full JSON dump\n found_comment_with_all_keywords = False\n for comment in comment_list:\n # Extract text via ADF\n adf_texts = extract_adf_text(comment)\n combined_adf = ' '.join(adf_texts).lower()\n\n # Also get full JSON dump as fallback\n full_dump = json.dumps(comment).lower()\n\n # Also check plain string body\n body_text = ''\n if isinstance(comment, dict):\n body_val = comment.get('body', '')\n if isinstance(body_val, str):\n body_text = body_val.lower()\n\n # Combine all text sources\n all_text = combined_adf + ' ' + full_dump + ' ' + body_text\n\n missing = [kw for kw in keywords if kw.lower() not in all_text]\n if not missing:\n found_comment_with_all_keywords = True\n break\n\n if found_comment_with_all_keywords:\n details.append(f'{issue_id}: found comment containing all required keywords {keywords}')\n else:\n failures.append(f'{issue_id}: no single comment contains all required keywords {keywords}. Checked {len(comment_list)} comment(s).')\n\n if failures:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': 'Failures: ' + '; '.join(failures) + ((' | Passes: ' + '; '.join(details)) if details else '')\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': 'All checks passed. ' + '; '.join(details)\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "b607ef6c-73ef-4e16-8c21-366d6d96d4df",
|
|
"sort_order": 9,
|
|
"rubric_text": "In jira_state.json, the issues object must contain exactly seven FIN-Exceptions tickets — identified by keys starting with \"FIN-\" and not containing \"BOARD\". One ticket must exist for each of: OPS-4 (missing Slack approval), OPS-5 (missing PDF), OPS-6 (GL conflict), OPS-9 (missing Slack approval), OPS-10 (GL conflict), OPS-11 (missing Slack approval), and OPS-12 (self-approval). Each ticket must reference its corresponding SUSP ID somewhere in its content.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n # The rubric says: \"the issues object must contain exactly seven FIN-Exceptions tickets\n # identified by keys starting with FIN- and not containing BOARD\"\n # So we look for jira_data[\"issues\"] as a dict keyed by ticket IDs.\n issues = None\n if isinstance(jira_data, dict):\n # Try to find an \"issues\" key\n if 'issues' in jira_data:\n issues = jira_data['issues']\n else:\n # Maybe the top-level dict itself is the issues dict\n issues = jira_data\n\n if issues is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find issues object in jira_state.json'}\n\n # Collect FIN- tickets that don't contain BOARD in their key\n fin_exception_tickets = {}\n\n if isinstance(issues, dict):\n for key, val in issues.items():\n key_upper = key.upper()\n if key_upper.startswith('FIN-') and 'BOARD' not in key_upper:\n fin_exception_tickets[key] = val\n elif isinstance(issues, list):\n # If issues is a list, look for key/id fields\n for item in issues:\n if isinstance(item, dict):\n key = str(item.get('key', '') or item.get('id', '') or '')\n key_upper = key.upper()\n if key_upper.startswith('FIN-') and 'BOARD' not in key_upper:\n fin_exception_tickets[key] = item\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'issues object is of unexpected type: {type(issues)}'}\n\n # Check count\n num_tickets = len(fin_exception_tickets)\n failures = []\n if num_tickets != 7:\n failures.append(f'Expected exactly 7 FIN-Exception tickets (keys starting with FIN- and not containing BOARD), found {num_tickets}. Keys found: {list(fin_exception_tickets.keys())}')\n\n # Required SUSP IDs\n required_ids = ['OPS-4', 'OPS-5', 'OPS-6', 'OPS-9', 'OPS-10', 'OPS-11', 'OPS-12']\n\n # For each required SUSP ID, check that at least one FIN-Exception ticket references it\n found_ids = set()\n for rid in required_ids:\n for key, val in fin_exception_tickets.items():\n # Serialize the entire ticket value to search for the SUSP ID\n ticket_str = json.dumps(val) if not isinstance(val, str) else val\n # Also check the key itself\n full_str = key + ' ' + ticket_str\n if rid in full_str:\n found_ids.add(rid)\n break\n\n missing_ids = [rid for rid in required_ids if rid not in found_ids]\n if missing_ids:\n failures.append(f'Missing FIN-Exception tickets referencing these SUSP IDs: {missing_ids}')\n\n if failures:\n score = len(found_ids) / len(required_ids) if not (num_tickets != 7 and len(found_ids) == 0) else 0.0\n return {'pass': False, 'score': score, 'feedback': 'Failures: ' + '; '.join(failures)}\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found exactly 7 FIN-Exception tickets with keys {list(fin_exception_tickets.keys())}, covering all required SUSP IDs: {required_ids}'\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "e720abd4-86ab-4a80-80b3-74074d7a2b1e",
|
|
"sort_order": 10,
|
|
"rubric_text": "In jira_state.json, all FIN-Exceptions tickets (for OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, OPS-12) must be assigned to jennifer.walsh.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef _flatten_issues(jira_data):\n \"\"\"Extract all individual issue dicts from the jira_state structure.\"\"\"\n all_issues = []\n if not isinstance(jira_data, dict):\n if isinstance(jira_data, list):\n return [i for i in jira_data if isinstance(i, dict)]\n return []\n\n for key, val in jira_data.items():\n if key == 'comments':\n continue\n if isinstance(val, list):\n for item in val:\n if isinstance(item, dict):\n all_issues.append(item)\n elif isinstance(val, dict):\n if any(k in val for k in ('key', 'fields', 'id', 'assignee', 'summary')):\n all_issues.append(val)\n else:\n for sub_key, sub_val in val.items():\n if isinstance(sub_val, dict):\n all_issues.append(sub_val)\n return all_issues\n\n\ndef _check_assignee_for_jennifer(ticket):\n \"\"\"Check ONLY the assignee field of a ticket for jennifer.walsh.\"\"\"\n jennifer_variants = ['jennifer.walsh', 'jennifer walsh', 'jennifer_walsh', 'jwalsh', 'j.walsh']\n\n if not isinstance(ticket, dict):\n return False\n\n containers = [ticket]\n if isinstance(ticket.get('fields'), dict):\n containers.append(ticket['fields'])\n\n for container in containers:\n for field_name in ['assignee', 'assigned_to', 'owner']:\n val = container.get(field_name)\n if val is None:\n continue\n if isinstance(val, str):\n if any(v in val.lower() for v in jennifer_variants):\n return True\n elif isinstance(val, dict):\n for subfield in ['accountId', 'displayName', 'emailAddress', 'name', 'key']:\n subval = val.get(subfield, '')\n if subval and isinstance(subval, str):\n if any(v in subval.lower() for v in jennifer_variants):\n return True\n return False\n\n\ndef _get_assignee_display(ticket):\n \"\"\"Extract a human-readable assignee string for error messages.\"\"\"\n if not isinstance(ticket, dict):\n return 'unknown'\n for container in [ticket.get('fields', {}), ticket]:\n if not isinstance(container, dict):\n continue\n for field_name in ['assignee', 'assigned_to', 'owner']:\n val = container.get(field_name)\n if val is None:\n continue\n if isinstance(val, dict):\n return val.get('displayName', val.get('accountId', str(val)))\n if isinstance(val, str) and val:\n return val\n return 'unknown'\n\n\ndef _get_issue_key(issue):\n \"\"\"Return the issue key (e.g. 'FIN-3') if present.\"\"\"\n if not isinstance(issue, dict):\n return ''\n key = issue.get('key')\n if isinstance(key, str):\n return key\n fields = issue.get('fields')\n if isinstance(fields, dict):\n k = fields.get('key')\n if isinstance(k, str):\n return k\n return ''\n\n\ndef _get_project_key(issue):\n \"\"\"Return the project key (e.g. 'FIN') if present.\"\"\"\n if not isinstance(issue, dict):\n return ''\n fields = issue.get('fields') if isinstance(issue.get('fields'), dict) else {}\n proj = fields.get('project') if isinstance(fields, dict) else None\n if isinstance(proj, dict):\n pk = proj.get('key')\n if isinstance(pk, str):\n return pk\n return ''\n\n\ndef _summary_and_description_text(issue):\n \"\"\"Return summary + description text for matching SUSP-IDs against, lowercased.\"\"\"\n if not isinstance(issue, dict):\n return ''\n fields = issue.get('fields') if isinstance(issue.get('fields'), dict) else issue\n parts = []\n summary = fields.get('summary')\n if isinstance(summary, str):\n parts.append(summary)\n desc = fields.get('description')\n if isinstance(desc, str):\n parts.append(desc)\n elif isinstance(desc, dict):\n # Atlassian Document Format — just dump it; SUSP-XXX will appear as a substring\n parts.append(json.dumps(desc))\n return ' '.join(parts)\n\n\ndef verify(workspace_path, external_services_path=None):\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n\n all_issues = _flatten_issues(jira_data)\n\n # Restrict to FIN-N tickets only (these are the exception tickets the agent creates).\n # Exclude SUSP-* sub-tasks regardless of their labels.\n fin_tickets = []\n for issue in all_issues:\n ikey = _get_issue_key(issue)\n pkey = _get_project_key(issue)\n if ikey.startswith('FIN-') or pkey == 'FIN':\n # Make sure we don't accidentally include something like 'FIN-Exceptions'\n # as a key — real keys look like FIN-<number>.\n tail = ikey.split('-', 1)[1] if '-' in ikey else ''\n if tail.isdigit() or pkey == 'FIN':\n fin_tickets.append(issue)\n\n required_ids = ['OPS-4', 'OPS-5', 'OPS-6', 'OPS-9',\n 'OPS-10', 'OPS-11', 'OPS-12']\n failures = []\n found_tickets = []\n\n for rid in required_ids:\n rid_lower = rid.lower()\n # Find FIN tickets whose summary/description references this SUSP id\n matching = [t for t in fin_tickets\n if rid_lower in _summary_and_description_text(t).lower()]\n\n if not matching:\n failures.append(f'{rid}: no FIN-* exception ticket found referencing this ID')\n continue\n\n # If multiple FIN tickets reference the same SUSP id, require that at least one\n # is assigned to jennifer.walsh. (Adjust to \"all\" if the criterion demands it.)\n ok_ticket = next((t for t in matching if _check_assignee_for_jennifer(t)), None)\n\n if ok_ticket is not None:\n found_tickets.append(rid)\n else:\n # Report the first matching ticket's assignee for context\n t = matching[0]\n assignee_info = _get_assignee_display(t)\n tkey = _get_issue_key(t) or '(unknown key)'\n failures.append(f'{rid}: FIN ticket {tkey} assignee is \"{assignee_info}\", not jennifer.walsh')\n\n if failures:\n score = len(found_tickets) / len(required_ids)\n return {\n 'pass': False,\n 'score': score,\n 'feedback': (f'Found {len(found_tickets)}/{len(required_ids)} correctly assigned. '\n f'Failures: ' + '; '.join(failures))\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': (f\"All {len(required_ids)} FIN-Exception tickets \"\n f\"({', '.join(required_ids)}) are assigned to jennifer.walsh\")\n }\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error processing jira_state.json: {e}'}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "eb714737-6b44-4948-b983-7d1f811133c5",
|
|
"sort_order": 11,
|
|
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-4, OPS-5, OPS-9, OPS-11, and OPS-12 must each have at least one comment in that list containing an escalation reason. Comments are stored in Atlassian doc format: extract text by walking body → content[] → content[] → text fields. Acceptable keywords indicating an escalation reason include: escalat, exception, missing, conflict, approval, pdf, slack, self, manager.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef _extract_adf_text(node):\n \"\"\"Recursively extract text from Atlassian Document Format nodes.\"\"\"\n texts = []\n if isinstance(node, dict):\n if 'text' in node:\n texts.append(str(node['text']))\n for key in ('body', 'content', 'children'):\n child = node.get(key)\n if isinstance(child, list):\n for item in child:\n texts.extend(_extract_adf_text(item))\n elif isinstance(child, dict):\n texts.extend(_extract_adf_text(child))\n elif isinstance(node, list):\n for item in node:\n texts.extend(_extract_adf_text(item))\n return texts\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n if not isinstance(jira_data, dict):\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json top-level is not a dict'}\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n comments_obj = jira_data.get('comments', {})\n if not isinstance(comments_obj, dict):\n # Fallback: maybe the entire jira_data acts as a container; try to find comments elsewhere\n comments_obj = {}\n\n # Also collect all issues from other top-level keys (issues list, etc.) for fallback\n all_issues = []\n for key, val in jira_data.items():\n if key == 'comments':\n continue\n if isinstance(val, list):\n all_issues.extend(val)\n elif isinstance(val, dict):\n all_issues.append(val)\n\n required_ids = ['OPS-4', 'OPS-5', 'OPS-9', 'OPS-11', 'OPS-12']\n escalation_keywords = ['escalat', 'exception', 'missing', 'conflict', 'approval', 'pdf', 'slack', 'self', 'manager']\n\n failures = []\n successes = []\n\n for tid in required_ids:\n combined_text = ''\n\n # Primary path: top-level comments object keyed by issue ID\n comment_list = comments_obj.get(tid, [])\n if not isinstance(comment_list, list):\n comment_list = [comment_list] if comment_list else []\n\n for comment in comment_list:\n # Extract text via ADF walking\n adf_texts = _extract_adf_text(comment)\n combined_text += ' '.join(adf_texts) + ' '\n # Also grab any plain string fields\n if isinstance(comment, dict):\n for field in ('body', 'text', 'content'):\n v = comment.get(field)\n if isinstance(v, str):\n combined_text += v + ' '\n elif isinstance(comment, str):\n combined_text += comment + ' '\n\n # Fallback: look through issues for matching issue and its comments\n if not combined_text.strip():\n for issue in all_issues:\n if not isinstance(issue, dict):\n continue\n issue_str = json.dumps(issue)\n if tid in issue_str:\n # Try to get comments from the issue object\n issue_comments = issue.get('comments', issue.get('comment', []))\n if not isinstance(issue_comments, list):\n issue_comments = [issue_comments] if issue_comments else []\n for comment in issue_comments:\n adf_texts = _extract_adf_text(comment)\n combined_text += ' '.join(adf_texts) + ' '\n if isinstance(comment, dict):\n for field in ('body', 'text', 'content'):\n v = comment.get(field)\n if isinstance(v, str):\n combined_text += v + ' '\n elif isinstance(comment, str):\n combined_text += comment + ' '\n # Also check description/notes as last resort\n if not combined_text.strip():\n desc = str(issue.get('description', '') or '') + ' ' + str(issue.get('notes', '') or '')\n combined_text += desc\n break\n\n combined_lower = combined_text.lower().strip()\n\n if not combined_lower:\n failures.append(f'{tid}: no comments found in jira_state.json comments object or issue data')\n continue\n\n has_escalation = any(kw in combined_lower for kw in escalation_keywords)\n if has_escalation:\n matched = [kw for kw in escalation_keywords if kw in combined_lower]\n # For OPS-12 specifically, require a negative indicator to distinguish\n # escalation comments (\"approval insufficient\") from clearing comments\n # (\"approval found\"). This prevents false positives when models clear\n # OPS-12 instead of excepting it.\n if tid == 'OPS-12':\n negative_indicators = ['missing', 'insufficient', 'invalid', 'not a manager', 'not manager', 'escalat', 'exception', 'unauthorized', 'junior', 'self-approv', 'improper', 'flag', 'issue', 'concern', 'problem', 'must be']\n has_negative = any(neg in combined_lower for neg in negative_indicators)\n if not has_negative:\n failures.append(f'{tid}: comment mentions escalation keywords {matched} but lacks a negative indicator — may be a clearing comment, not an escalation')\n else:\n successes.append(f'{tid}: found escalation keywords {matched} with negative indicator')\n else:\n successes.append(f'{tid}: found escalation keywords {matched}')\n else:\n failures.append(f'{tid}: comment text found but no escalation keyword detected. Text snippet: \"{combined_lower[:300]}\"')\n\n score = len(successes) / len(required_ids)\n\n if failures:\n feedback = 'Failures: ' + '; '.join(failures)\n if successes:\n feedback += ' | Successes: ' + '; '.join(successes)\n return {'pass': False, 'score': score, 'feedback': feedback}\n\n return {'pass': True, 'score': 1.0, 'feedback': 'All required subtasks (OPS-4, OPS-5, OPS-9, OPS-11, OPS-12) have comments noting escalation reasons. Details: ' + '; '.join(successes)}\n",
|
|
"criterion_type": "expected_output"
|
|
}
|
|
]
|