87 lines
68 KiB
JSON
87 lines
68 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "cd482ae3-5a98-473d-944b-90077a455e30",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "File `Suspense_Reconciliation_March2026.xlsx` must exist in the workspace and contain columns: Support Found, Support Location, Conflict, Status, and Resolution Notes.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip() if cell.value else '' for cell in ws[1]]\n headers_lower = [h.lower() for h in headers]\n required_cols = ['support found', 'support location', 'conflict', 'status', 'resolution notes']\n missing = [col for col in required_cols if col not in headers_lower]\n if missing:\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns: {missing}. Found columns: {headers}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'File exists and contains all required columns: {required_cols}'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading file: {e}'}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "294d7b7f-70c3-46e0-993b-71b9f0cecf2c",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, cleared items OPS-2 ($3,250), OPS-3 ($7,800), OPS-7 ($2,100), and OPS-8 ($15,000) must each show Status=FIN-100, Support Found=Y, Conflict=N, and non-empty Resolution Notes.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n def col_idx(name):\n for i, h in enumerate(headers):\n if name.lower() in h:\n return i\n return None\n id_col = col_idx('susp') if col_idx('susp') is not None else col_idx('id') if col_idx('id') is not None else 0\n # Try to find transaction ID column\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or (i > 0 and 'support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h == 'support found':\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [status_col, support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find required columns. Headers: {headers}'}\n target_ids = {'OPS-2', 'OPS-3', 'OPS-7', 'OPS-8'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n status = str(row[status_col]).strip() if row[status_col] else ''\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip() if row[notes_col] else ''\n if 'FIN-100' not in status.upper() and status.upper() != 'FIN-100':\n failures.append(f'{tid}: Status={status} (expected FIN-100)')\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'{tid}: Support Found={sf} (expected Y)')\n if conflict not in ('N', 'NO', 'FALSE', '0'):\n failures.append(f'{tid}: Conflict={conflict} (expected N)')\n if not notes or notes.lower() == 'none':\n failures.append(f'{tid}: Resolution Notes is empty')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': f'All cleared items (OPS-2, OPS-3, OPS-7, OPS-8) show Status=FIN-100, Support Found=Y, Conflict=N, and non-empty Resolution Notes'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "731b9a6e-3474-439c-95c7-5855d1a54007",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-3 ($7,800) and OPS-8 ($15,000) Resolution Notes must reference a Slack approval message.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if notes_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Resolution Notes column. Headers: {headers}'}\n target_ids = {'OPS-3', 'OPS-8'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n slack_keywords = ['slack', 'approval', 'approved', '#finance', 'channel', 'message']\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if not any(kw in notes for kw in slack_keywords):\n failures.append(f'{tid}: Resolution Notes does not reference Slack approval. Notes: \"{notes}\"')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'OPS-3 and OPS-8 Resolution Notes reference Slack approval message'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "753ac24a-0b95-46ad-9b11-a3a2dd24d630",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, exception items OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, and OPS-12 must each show Status=EXCEPTION and must NOT show Status=FIN-100.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n if status_col is None:\n return {'pass': False, 'score': 0.0, 'feedback': f'Could not find Status column. Headers: {headers}'}\n target_ids = {'OPS-4', 'OPS-5', 'OPS-6', 'OPS-9', 'OPS-10', 'OPS-11', 'OPS-12'}\n found = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in target_ids:\n found[row_id] = row\n failures = []\n for tid in target_ids:\n if tid not in found:\n failures.append(f'{tid}: not found in workbook')\n continue\n row = found[tid]\n status = str(row[status_col]).strip().upper() if row[status_col] else ''\n if 'EXCEPTION' not in status:\n failures.append(f'{tid}: Status={status} (expected EXCEPTION)')\n if 'FIN-100' in status:\n failures.append(f'{tid}: Status={status} must NOT be FIN-100')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'Failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'All exception items (OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, OPS-12) show Status=EXCEPTION'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "e23f7481-1554-4d6d-9055-770e06854a17",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In file Suspense_Reconciliation_March2026.xlsx, OPS-4 must show Support Found=Y, Conflict=N, and Resolution Notes indicating missing Slack approval. OPS-5 must show Support Found=N, Conflict=N, and Resolution Notes indicating missing PDF. OPS-11 must show Support Found=Y, Conflict=N, and Resolution Notes indicating no manager approval found in #finance-approvals. OPS-12 must show Support Found=Y, Conflict=N, and Resolution Notes indicating approval must be confirmed by a manager, not a junior analyst.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Try to find the file - be flexible\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n candidates = list(Path(workspace_path).glob('*Suspense*Reconciliation*March*2026*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*reconciliation*march*2026*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*Suspense*March*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*march*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*Suspense*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*suspense*.xlsx'))\n if not candidates:\n candidates = list(Path(workspace_path).glob('*.xlsx'))\n if candidates:\n xlsx_path = candidates[0]\n else:\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found in workspace'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = None\n for sheet_name in wb.sheetnames:\n candidate_ws = wb[sheet_name]\n first_row = list(candidate_ws.iter_rows(min_row=1, max_row=1, values_only=True))\n if not first_row:\n continue\n candidate_headers = [str(cell).strip().lower() if cell else '' for cell in first_row[0]]\n has_support = any('support' in h for h in candidate_headers)\n has_conflict = any('conflict' in h for h in candidate_headers)\n has_notes = any('resolution' in h or 'notes' in h for h in candidate_headers)\n if has_support and has_conflict and has_notes:\n ws = candidate_ws\n break\n if ws is None:\n ws = wb.active\n\n first_row = list(ws.iter_rows(min_row=1, max_row=1, values_only=True))\n if not first_row or not first_row[0]:\n return {'pass': False, 'score': 0.0, 'feedback': 'No headers found in first row'}\n headers = [str(cell).strip().lower() if cell else '' for cell in first_row[0]]\n\n id_col = None\n for i, h in enumerate(headers):\n if 'susp' in h or h == 'id' or 'item id' in h or 'item_id' in h:\n id_col = i\n break\n if id_col is None:\n for i, h in enumerate(headers):\n if 'item' in h or 'transaction' in h or 'entry' in h or 'ref' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if 'support' in h and 'location' not in h and 'loc' not in h:\n support_found_col = i\n break\n\n support_location_col = None\n for i, h in enumerate(headers):\n if 'support location' in h or ('support' in h and 'location' in h) or ('support' in h and 'loc' in h):\n support_location_col = i\n break\n\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n\n status_col = None\n for i, h in enumerate(headers):\n if h == 'status' or 'status' in h:\n status_col = i\n break\n\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h and 'note' in h:\n notes_col = i\n break\n if notes_col is None
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "8c35192a-a420-4647-9f7b-a056ddad0a04",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-6 must show Support Found=Y, Conflict=Y, and Resolution Notes documenting both GL codes (Invoice: 7100-Capital-Equipment and Excel: 8100-IT-Equipment). OPS-10 must show Support Found=Y, Conflict=Y, and Resolution Notes documenting both GL codes (Invoice: 6200-Office-Supplies and Excel: 6200-Office-Exp).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h.startswith('support'):\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns. Headers: {headers}'}\n rows = {}\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id in ('OPS-6', 'OPS-10'):\n rows[row_id] = row\n failures = []\n # OPS-6: Support Found=Y, Conflict=Y, notes with 7100 and 8100\n if 'OPS-6' not in rows:\n failures.append('OPS-6: not found')\n else:\n row = rows['OPS-6']\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-6: Support Found={sf} (expected Y)')\n if conflict not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-6: Conflict={conflict} (expected Y)')\n has_7100 = '7100' in notes or 'capital' in notes\n has_8100 = '8100' in notes or 'it' in notes or 'equipment' in notes\n if not has_7100:\n failures.append(f'OPS-6: Resolution Notes missing GL code 7100/Capital-Equipment: \"{notes}\"')\n if not has_8100:\n failures.append(f'OPS-6: Resolution Notes missing GL code 8100/IT-Equipment: \"{notes}\"')\n # OPS-10: Support Found=Y, Conflict=Y, notes with 6200-Office-Supplies and 6200-Office-Exp\n if 'OPS-10' not in rows:\n failures.append('OPS-10: not found')\n else:\n row = rows['OPS-10']\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-10: Support Found={sf} (expected Y)')\n if conflict not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'OPS-10: Conflict={conflict} (expected Y)')\n has_6200 = '6200' in notes\n has_office = 'office' in notes\n if not has_6200:\n failures.append(f'OPS-10: Resolution Notes missing GL code 6200:
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "a5a058cc-1ac7-466b-8799-e7bc4162bba9",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In file `Suspense_Reconciliation_March2026.xlsx`, OPS-9 must show Support Found=Y, Conflict=N, Status=EXCEPTION, and Resolution Notes indicating missing Slack approval.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n xlsx_path = Path(workspace_path) / 'Suspense_Reconciliation_March2026.xlsx'\n if not xlsx_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'File Suspense_Reconciliation_March2026.xlsx not found'}\n try:\n wb = openpyxl.load_workbook(xlsx_path)\n ws = wb.active\n headers = [str(cell.value).strip().lower() if cell.value else '' for cell in ws[1]]\n id_col = None\n for i, h in enumerate(headers):\n if 'id' in h or 'susp' in h or 'transaction' in h or 'item' in h:\n id_col = i\n break\n if id_col is None:\n id_col = 0\n status_col = None\n for i, h in enumerate(headers):\n if 'status' in h:\n status_col = i\n break\n support_found_col = None\n for i, h in enumerate(headers):\n if 'support found' in h or ('support' in h and 'found' in h):\n support_found_col = i\n break\n if support_found_col is None:\n for i, h in enumerate(headers):\n if h.startswith('support'):\n support_found_col = i\n break\n conflict_col = None\n for i, h in enumerate(headers):\n if 'conflict' in h:\n conflict_col = i\n break\n notes_col = None\n for i, h in enumerate(headers):\n if 'resolution' in h or 'notes' in h:\n notes_col = i\n break\n if any(c is None for c in [status_col, support_found_col, conflict_col, notes_col]):\n return {'pass': False, 'score': 0.0, 'feedback': f'Missing required columns. Headers: {headers}'}\n susp_008_row = None\n for row in ws.iter_rows(min_row=2, values_only=True):\n row_id = str(row[id_col]).strip() if row[id_col] else ''\n if row_id == 'OPS-9':\n susp_008_row = row\n break\n if susp_008_row is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'OPS-9 not found in workbook'}\n row = susp_008_row\n sf = str(row[support_found_col]).strip().upper() if row[support_found_col] else ''\n conflict = str(row[conflict_col]).strip().upper() if row[conflict_col] else ''\n status = str(row[status_col]).strip().upper() if row[status_col] else ''\n notes = str(row[notes_col]).strip().lower() if row[notes_col] else ''\n failures = []\n if sf not in ('Y', 'YES', 'TRUE', '1'):\n failures.append(f'Support Found={sf} (expected Y)')\n if conflict not in ('N', 'NO', 'FALSE', '0'):\n failures.append(f'Conflict={conflict} (expected N)')\n if 'EXCEPTION' not in status:\n failures.append(f'Status={status} (expected EXCEPTION)')\n if not any(kw in notes for kw in ['slack', 'approval', 'missing']):\n failures.append(f'Resolution Notes does not indicate missing Slack approval: \"{notes}\"')\n if failures:\n return {'pass': False, 'score': 0.0, 'feedback': 'OPS-9 failures: ' + '; '.join(failures)}\n return {'pass': True, 'score': 1.0, 'feedback': 'OPS-9 shows Support Found=Y, Conflict=N, Status=EXCEPTION, and Resolution Notes indicating missing Slack approval'}\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error: {e}'}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "5f6dad05-f5cf-4e49-9cb1-0f36e9672343",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-2, OPS-3, OPS-7, and OPS-8 must each have at least one comment in that list containing the transaction ID, the target GL code, and the phrase \"FIN-100 Cleared & Posted\" (case-insensitive).ve).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n # Look for jira_state.json in external_services_path first, then workspace\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in workspace or external_services_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n target_ids = ['OPS-2', 'OPS-3', 'OPS-7', 'OPS-8']\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n # So we expect jira_data to have a top-level key \"comments\" which is a dict keyed by issue ID\n # e.g. { \"comments\": { \"OPS-2\": [...], \"OPS-3\": [...], ... }, ... }\n \n # Try to find the comments object\n comments_obj = None\n if isinstance(jira_data, dict):\n # Direct top-level \"comments\" key\n if 'comments' in jira_data:\n comments_obj = jira_data['comments']\n # Maybe nested under another key\n if comments_obj is None:\n for k, v in jira_data.items():\n if isinstance(v, dict) and any(tid in v or tid.lower() in str(v).lower() for tid in target_ids):\n # Check if this looks like a comments-by-issue-id dict\n if any(isinstance(vv, list) for vv in v.values()):\n comments_obj = v\n break\n \n # Fallback: search the entire structure for comment lists\n if comments_obj is None:\n # Try to find any dict that has issue IDs as keys mapping to lists\n def find_comments_dict(obj, depth=0):\n if depth > 5:\n return None\n if isinstance(obj, dict):\n # Check if this dict itself has target IDs as keys mapping to lists\n matching = sum(1 for tid in target_ids if tid in obj and isinstance(obj.get(tid), list))\n if matching >= 1:\n return obj\n for k, v in obj.items():\n result = find_comments_dict(v, depth + 1)\n if result is not None:\n return result\n return None\n comments_obj = find_comments_dict(jira_data)\n\n if comments_obj is None or not isinstance(comments_obj, dict):\n return {'pass': False, 'score': 0.0, 'feedback': f'No top-level comments object found in jira_state.json. Top-level keys: {list(jira_data.keys()) if isinstance(jira_data, dict) else \"not a dict\"}'}\n\n failures = []\n successes = []\n\n for tid in target_ids:\n tid_lower = tid.lower()\n\n # Find the comment list for this issue ID (case-insensitive key match)\n comment_list = None\n for key, val in comments_obj.items():\n if key.lower() == tid_lower or key.upper() == tid.upper():\n comment_list = val\n break\n \n if comment_list is None:\n # Also try looking inside issues structure for comments\n failures.append(f'{tid}: no comment list found keyed by this issue ID in the comments object')\n continue\n\n if not isinstance(comment_list, list):\n comment_list = [comment_list]\n\n # Check each comment for required content:\n # 1. transaction ID (the SUSP-XXX)\n # 2. target GL code (some numeric code or GL reference)\n # 3. phrase \"FIN-100 Cleared & Posted\" (case-insensitive)\n found_valid_comment = False\n best_comment_
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "2e4c68b9-6fc5-4ca9-9c09-bc2f3cb0ddf4",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-6 must have at least one comment containing both GL codes \"7100\" and \"8100\". OPS-10 must have at least one comment containing both \"6200\" and \"office\". Comments are stored in Atlassian doc format: extract text by walking body → content[] → content[] → text fields.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef extract_adf_text(node):\n \"\"\"Recursively extract text from Atlassian Document Format nodes.\"\"\"\n texts = []\n if isinstance(node, dict):\n if 'text' in node:\n texts.append(str(node['text']))\n for key in ('content', 'body'):\n if key in node:\n val = node[key]\n if isinstance(val, list):\n for item in val:\n texts.extend(extract_adf_text(item))\n elif isinstance(val, dict):\n texts.extend(extract_adf_text(val))\n elif isinstance(node, list):\n for item in node:\n texts.extend(extract_adf_text(item))\n return texts\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / 'jira_state.json')\n candidates.append(Path(workspace_path) / 'jira_state.json')\n for c in candidates:\n if c.exists():\n jira_path = c\n break\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n # So we look for jira_data[\"comments\"] as a dict keyed by issue ID like \"OPS-6\"\n comments_obj = None\n if isinstance(jira_data, dict):\n # Try direct top-level \"comments\" key\n if 'comments' in jira_data and isinstance(jira_data['comments'], dict):\n comments_obj = jira_data['comments']\n\n # Fallback: search the entire structure for a dict that has keys matching SUSP-xxx\n if comments_obj is None and isinstance(jira_data, dict):\n for key, val in jira_data.items():\n if isinstance(val, dict):\n # Check if any sub-key looks like an issue ID\n for subkey in val:\n if isinstance(subkey, str) and subkey.upper().startswith('SUSP-'):\n comments_obj = val\n break\n if comments_obj:\n break\n\n checks = {\n 'OPS-6': ['7100', '8100'],\n 'OPS-10': ['6200', 'office']\n }\n failures = []\n details = []\n\n for issue_id, keywords in checks.items():\n # Try to find comments for this issue\n comment_list = None\n if comments_obj is not None:\n # Try exact key and case-insensitive\n for k, v in comments_obj.items():\n if k.upper() == issue_id.upper():\n comment_list = v if isinstance(v, list) else [v]\n break\n\n # Fallback: look through issues list for embedded comments\n if comment_list is None and isinstance(jira_data, dict):\n issues = []\n if 'issues' in jira_data and isinstance(jira_data['issues'], list):\n issues = jira_data['issues']\n elif 'subtasks' in jira_data and isinstance(jira_data['subtasks'], list):\n issues = jira_data['subtasks']\n for issue in issues:\n if isinstance(issue, dict):\n issue_str = json.dumps(issue).lower()\n if issue_id.lower() in issue_str:\n if 'comments' in issue:\n comment_list = issue['comments'] if isinstance(issue['comments'], list) else [issue['comments']]\n elif 'comment' in issue:\n c = issue['comment']\n comment_list = c if isinstance(c, list) else [c]\n break\n\n if commen
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "b607ef6c-73ef-4e16-8c21-366d6d96d4df",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "In jira_state.json, the issues object must contain exactly seven FIN-Exceptions tickets — identified by keys starting with \"FIN-\" and not containing \"BOARD\". One ticket must exist for each of: OPS-4 (missing Slack approval), OPS-5 (missing PDF), OPS-6 (GL conflict), OPS-9 (missing Slack approval), OPS-10 (GL conflict), OPS-11 (missing Slack approval), and OPS-12 (self-approval). Each ticket must reference its corresponding SUSP ID somewhere in its content.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n # The rubric says: \"the issues object must contain exactly seven FIN-Exceptions tickets\n # identified by keys starting with FIN- and not containing BOARD\"\n # So we look for jira_data[\"issues\"] as a dict keyed by ticket IDs.\n issues = None\n if isinstance(jira_data, dict):\n # Try to find an \"issues\" key\n if 'issues' in jira_data:\n issues = jira_data['issues']\n else:\n # Maybe the top-level dict itself is the issues dict\n issues = jira_data\n\n if issues is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'Could not find issues object in jira_state.json'}\n\n # Collect FIN- tickets that don't contain BOARD in their key\n fin_exception_tickets = {}\n\n if isinstance(issues, dict):\n for key, val in issues.items():\n key_upper = key.upper()\n if key_upper.startswith('FIN-') and 'BOARD' not in key_upper:\n fin_exception_tickets[key] = val\n elif isinstance(issues, list):\n # If issues is a list, look for key/id fields\n for item in issues:\n if isinstance(item, dict):\n key = str(item.get('key', '') or item.get('id', '') or '')\n key_upper = key.upper()\n if key_upper.startswith('FIN-') and 'BOARD' not in key_upper:\n fin_exception_tickets[key] = item\n else:\n return {'pass': False, 'score': 0.0, 'feedback': f'issues object is of unexpected type: {type(issues)}'}\n\n # Check count\n num_tickets = len(fin_exception_tickets)\n failures = []\n if num_tickets != 7:\n failures.append(f'Expected exactly 7 FIN-Exception tickets (keys starting with FIN- and not containing BOARD), found {num_tickets}. Keys found: {list(fin_exception_tickets.keys())}')\n\n # Required SUSP IDs\n required_ids = ['OPS-4', 'OPS-5', 'OPS-6', 'OPS-9', 'OPS-10', 'OPS-11', 'OPS-12']\n\n # For each required SUSP ID, check that at least one FIN-Exception ticket references it\n found_ids = set()\n for rid in required_ids:\n for key, val in fin_exception_tickets.items():\n # Serialize the entire ticket value to search for the SUSP ID\n ticket_str = json.dumps(val) if not isinstance(val, str) else val\n # Also check the key itself\n full_str = key + ' ' + ticket_str\n if rid in full_str:\n found_ids.add(rid)\n break\n\n missing_ids = [rid for rid in required_ids if rid not in found_ids]\n if missing_ids:\n failures.append(f'Missing FIN-Exception tickets referencing these SUSP IDs: {missing_ids}')\n\n if failures:\n score = len(found_ids) / len(required_ids) if not (num_tickets != 7 and len(found_ids) == 0) else 0.0\n return {'pass': False, 'score': score, 'feedback': 'Failures: ' + '; '.join(failures)}\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'Found exactly 7 FIN-Exception tickets with keys {list(fin_exception_tickets.keys())}, covering all required SUSP IDs: {required_ids}'\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "e720abd4-86ab-4a80-80b3-74074d7a2b1e",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "In jira_state.json, all FIN-Exceptions tickets (for OPS-4, OPS-5, OPS-6, OPS-9, OPS-10, OPS-11, OPS-12) must be assigned to jennifer.walsh.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef _flatten_issues(jira_data):\n \"\"\"Extract all individual issue dicts from the jira_state structure.\"\"\"\n all_issues = []\n if not isinstance(jira_data, dict):\n if isinstance(jira_data, list):\n return [i for i in jira_data if isinstance(i, dict)]\n return []\n\n for key, val in jira_data.items():\n if key == 'comments':\n continue\n if isinstance(val, list):\n for item in val:\n if isinstance(item, dict):\n all_issues.append(item)\n elif isinstance(val, dict):\n if any(k in val for k in ('key', 'fields', 'id', 'assignee', 'summary')):\n all_issues.append(val)\n else:\n for sub_key, sub_val in val.items():\n if isinstance(sub_val, dict):\n all_issues.append(sub_val)\n return all_issues\n\n\ndef _check_assignee_for_jennifer(ticket):\n \"\"\"Check ONLY the assignee field of a ticket for jennifer.walsh.\"\"\"\n jennifer_variants = ['jennifer.walsh', 'jennifer walsh', 'jennifer_walsh', 'jwalsh', 'j.walsh']\n\n if not isinstance(ticket, dict):\n return False\n\n containers = [ticket]\n if isinstance(ticket.get('fields'), dict):\n containers.append(ticket['fields'])\n\n for container in containers:\n for field_name in ['assignee', 'assigned_to', 'owner']:\n val = container.get(field_name)\n if val is None:\n continue\n if isinstance(val, str):\n if any(v in val.lower() for v in jennifer_variants):\n return True\n elif isinstance(val, dict):\n for subfield in ['accountId', 'displayName', 'emailAddress', 'name', 'key']:\n subval = val.get(subfield, '')\n if subval and isinstance(subval, str):\n if any(v in subval.lower() for v in jennifer_variants):\n return True\n return False\n\n\ndef _get_assignee_display(ticket):\n \"\"\"Extract a human-readable assignee string for error messages.\"\"\"\n if not isinstance(ticket, dict):\n return 'unknown'\n for container in [ticket.get('fields', {}), ticket]:\n if not isinstance(container, dict):\n continue\n for field_name in ['assignee', 'assigned_to', 'owner']:\n val = container.get(field_name)\n if val is None:\n continue\n if isinstance(val, dict):\n return val.get('displayName', val.get('accountId', str(val)))\n if isinstance(val, str) and val:\n return val\n return 'unknown'\n\n\ndef _get_issue_key(issue):\n \"\"\"Return the issue key (e.g. 'FIN-3') if present.\"\"\"\n if not isinstance(issue, dict):\n return ''\n key = issue.get('key')\n if isinstance(key, str):\n return key\n fields = issue.get('fields')\n if isinstance(fields, dict):\n k = fields.get('key')\n if isinstance(k, str):\n return k\n return ''\n\n\ndef _get_project_key(issue):\n \"\"\"Return the project key (e.g. 'FIN') if present.\"\"\"\n if not isinstance(issue, dict):\n return ''\n fields = issue.get('fields') if isinstance(issue.get('fields'), dict) else {}\n proj = fields.get('project') if isinstance(fields, dict) else None\n if isinstance(proj, dict):\n pk = proj.get('key')\n if isinstance(pk, str):\n return pk\n return ''\n\n\ndef _summary_and_description_text(issue):\n \"\"\"Return summary + description text for matching SUSP-IDs against, lowercased.\"\"\"\n if not isinstance(issue, dict):\n return ''\n fields = issue.get('fields') if isinstance(issue.get('fields'), dict) else issue\n parts = []\n summary = fields.get('summary')\n if isinstance(summary, str):\n parts.append(summary)\n desc = fields.get('description')\n if
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "eb714737-6b44-4948-b983-7d1f811133c5",
|
||
|
|
"sort_order": 11,
|
||
|
|
"rubric_text": "In jira_state.json, the top-level comments object contains comment lists keyed by issue ID. OPS-4, OPS-5, OPS-9, OPS-11, and OPS-12 must each have at least one comment in that list containing an escalation reason. Comments are stored in Atlassian doc format: extract text by walking body → content[] → content[] → text fields. Acceptable keywords indicating an escalation reason include: escalat, exception, missing, conflict, approval, pdf, slack, self, manager.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef _extract_adf_text(node):\n \"\"\"Recursively extract text from Atlassian Document Format nodes.\"\"\"\n texts = []\n if isinstance(node, dict):\n if 'text' in node:\n texts.append(str(node['text']))\n for key in ('body', 'content', 'children'):\n child = node.get(key)\n if isinstance(child, list):\n for item in child:\n texts.extend(_extract_adf_text(item))\n elif isinstance(child, dict):\n texts.extend(_extract_adf_text(child))\n elif isinstance(node, list):\n for item in node:\n texts.extend(_extract_adf_text(item))\n return texts\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n jira_path = None\n if external_services_path:\n candidate = Path(external_services_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n candidate = Path(workspace_path) / 'jira_state.json'\n if candidate.exists():\n jira_path = candidate\n if jira_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in external_services_path or workspace_path'}\n\n try:\n with open(jira_path) as f:\n jira_data = json.load(f)\n except Exception as e:\n return {'pass': False, 'score': 0.0, 'feedback': f'Error reading jira_state.json: {e}'}\n\n if not isinstance(jira_data, dict):\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json top-level is not a dict'}\n\n # The rubric says: \"the top-level comments object contains comment lists keyed by issue ID\"\n comments_obj = jira_data.get('comments', {})\n if not isinstance(comments_obj, dict):\n # Fallback: maybe the entire jira_data acts as a container; try to find comments elsewhere\n comments_obj = {}\n\n # Also collect all issues from other top-level keys (issues list, etc.) for fallback\n all_issues = []\n for key, val in jira_data.items():\n if key == 'comments':\n continue\n if isinstance(val, list):\n all_issues.extend(val)\n elif isinstance(val, dict):\n all_issues.append(val)\n\n required_ids = ['OPS-4', 'OPS-5', 'OPS-9', 'OPS-11', 'OPS-12']\n escalation_keywords = ['escalat', 'exception', 'missing', 'conflict', 'approval', 'pdf', 'slack', 'self', 'manager']\n\n failures = []\n successes = []\n\n for tid in required_ids:\n combined_text = ''\n\n # Primary path: top-level comments object keyed by issue ID\n comment_list = comments_obj.get(tid, [])\n if not isinstance(comment_list, list):\n comment_list = [comment_list] if comment_list else []\n\n for comment in comment_list:\n # Extract text via ADF walking\n adf_texts = _extract_adf_text(comment)\n combined_text += ' '.join(adf_texts) + ' '\n # Also grab any plain string fields\n if isinstance(comment, dict):\n for field in ('body', 'text', 'content'):\n v = comment.get(field)\n if isinstance(v, str):\n combined_text += v + ' '\n elif isinstance(comment, str):\n combined_text += comment + ' '\n\n # Fallback: look through issues for matching issue and its comments\n if not combined_text.strip():\n for issue in all_issues:\n if not isinstance(issue, dict):\n continue\n issue_str = json.dumps(issue)\n if tid in issue_str:\n # Try to get comments from the issue object\n issue_comments = issue.get('comments', issue.get('comment', []))\n if not isinstance(issue_comments, list):\n issue_comments = [issue_comments] if issue_comments else []\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
}
|
||
|
|
]
|