Add Handbook.md benchmark tasks
This commit is contained in:
commit
72f9a65917
1800 files changed
+323589
No files matched your search
@@ -0,0 +1,51 @@
|
||||
[
|
||||
{
|
||||
"id": "fd5fea82-8bfe-48e1-81d0-be8b87856e35",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `jira_state.json`, CLM_LIFE-1 must NOT have CLM-910 (Identity Verification Hold) or CLM-920 (Complex Beneficiary Hold) set.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n # --- Locate CLM_LIFE-1 ---\n issue = None\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n\n if isinstance(issues, dict):\n issue = issues.get(\"CLM_LIFE-1\")\n if issue is None:\n for key, val in issues.items():\n if key.upper() == \"CLM_LIFE-1\":\n issue = val\n break\n\n if issue is None and isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == \"CLM_LIFE-1\":\n issue = item\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-1 not found in jira_state.json\"}\n\n # --- Extract status name (handles nested dict or plain string) ---\n def get_status_name(issue):\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n status_field = fields.get(\"status\")\n if isinstance(status_field, dict):\n return str(status_field.get(\"name\", \"\")).strip()\n elif isinstance(status_field, str):\n return status_field.strip()\n top_status = issue.get(\"status\")\n if isinstance(top_status, dict):\n return str(top_status.get(\"name\", \"\")).strip()\n elif isinstance(top_status, str):\n return top_status.strip()\n return \"\"\n\n status_name = get_status_name(issue)\n status_lower = status_name.lower()\n\n # --- Extract labels ---\n labels = []\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n raw_labels = fields.get(\"labels\", [])\n if isinstance(raw_labels, list):\n labels = [str(l).lower() for l in raw_labels]\n\n # --- Check CLM-910 NOT set (status and labels only, NOT comment text) ---\n has_910 = (\n \"clm-910\" in status_lower\n or \"identity verification hold\" in status_lower\n or any(\"clm-910\" in lbl for lbl in labels)\n or any(\"identity-verification-hold\" in lbl or \"identity verification hold\" in lbl for lbl in labels)\n )\n\n # --- Check CLM-920 NOT set (status and labels only) ---\n has_920 = (\n \"clm-920\" in status_lower\n or \"complex beneficiary hold\" in status_lower\n or any(\"clm-920\" in lbl for lbl in labels)\n or any(\"complex-beneficiary-hold\" in lbl or \"complex beneficiary hold\" in lbl for lbl in labels)\n )\n\n # --- Check status is \"Route to AP for Payout\" ---\n routed = \"route to ap for payout\" in status_lower\n\n if not routed:\n transitions = issue.get(\"transitions\", [])\n if isinstance(transitions, list):\n for t in transitions:\n t_name = \"\"\n if isinstance(t, dict):\n t_name = str(t.get(\"name\", t.get(\"to\", \"\"))).lower()\n elif isinstance(t, str):\n t_name = t.lower()\n if \"route to ap for payout\" in t_name:\n routed = True\n break\n\n # --- Build result ---\n issues_found = []\n if has_910:\n issues_found.append(f\"CLM-910 Identity Verification Hold is set (status: '{status_name}')\")\n if has_920:\n issues_found.append(f\"CLM-920 Complex Beneficiary Hold is set (status: '{status_name}')\")\n if issues_found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-1: \" + \"; \".join(issues_found)}\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"CLM_LIFE-1: No CLM-910, no CLM-920\",\n }",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "3328f333-48c7-45b0-a940-a7e9e52f2b5f",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `jira_state.json`, CLM_LIFE-1 must have a structured beneficiary allocation comment containing Policy LC-2019-08844, Face Amount $400,000.00, Maria L. Martinez (Primary, 60%, $240,000.00), David R. Martinez (Primary, 40%, $160,000.00), with a header line and Analyst/Date attribution (\"analyst\" and the date must appear within 80 characters of each other), and must NOT include Elena Martinez (mention of her exclusion is acceptable).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n # 1. Top-level comments dict\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n # 2. Inline comments on the issue (fields.comment.comments)\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {}) if isinstance(issue, dict) else {}\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n # --- Locate CLM_LIFE-1 ---\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n issue = None\n if isinstance(issues, dict):\n issue = issues.get(\"CLM_LIFE-1\")\n if issue is None:\n for key, val in issues.items():\n if key.upper() == \"CLM_LIFE-1\":\n issue = val\n break\n if issue is None and isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == \"CLM_LIFE-1\":\n issue = item\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-1 not found in jira_state.json\"}\n\n # --- Get all comment text (ADF-aware, checks both top-level and inline) ---\n comments_text = get_all_comment_text(data, \"CLM_LIFE-1\")\n ct_lower = comments_text.lower()\n\n failures = []\n\n # Check header line: \"beneficiary allocation\" somewhere in the comment\n if \"beneficiary allocation\" not in ct_lower:\n failures.append(\"Missing 'Beneficiary Allocation' header line\")\n\n # Check policy LC-2019-08844\n if \"lc-2019-08844\" not in ct_lower:\n failures.append(\"Missing Policy LC-2019-08844\")\n\n # Check face amount $400,000.00\n if \"400,000\" not in comments_text and \"400000\" not in comments_text:\n failures.append(\"Missing Face Amount $400,000.00\")\n\n # Check Maria L. Martinez (Primary, 60%, $240,000.00)\n if \"maria\" not in ct_lower or \"martinez\" not in ct_lower:\n failures.append(\"Missing beneficiary Maria L. Martinez\")\n else:\n if \"240,000\" not in comments_text and \"240000\" not in comments_text:\n failures.append(\"Missing Maria Martinez amount $240,000.00\")\n if \"60%\" not in comments_text and \"60 %\" not in comments_text:\n failures.append(\"Missing Maria Martinez 60% split\")\n\n # Check David R. Martinez (Primary, 40%, $160,000.00)\n if \"david\" not in ct_lower:\n failures.append(\"Missing beneficiary David R. Martinez\")\n else:\n if \"160,000\" not in comments_text and \"160000\" not in comments_text:\n failures.append(\"Missing David Martinez amount $160,000.00\")\n if \"40%\" not in comments_text and \"40 %\" not in comments_text:\n failures.append(\"Missing David Martinez 40% split\")\n\n # Check Elena Martinez NOT included as a beneficiary in the allocation\n # Mention of her exclusion (e.g., \"Elena Martinez excluded\" or \"not included\") is acceptable.\n # We look for 'elena' in the allocation section and check if it's in the context of exclusion.\n alloc_start = ct_lower.find(\"beneficiary allocation\")\n if alloc_start != -1:\n alloc_section = ct_lower[alloc_start:]\n if \"elena\" in alloc_section:\n # Check if Elena is mentioned only in context of exclusion\n # Look for exclusion-relatLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "819c155d-7a13-4695-978e-f567c5be3cc9",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `jira_state.json`, CLM_LIFE-2 must NOT have CLM-910 or CLM-920 set, and must have a structured beneficiary allocation comment containing Policy LC-2021-15290, Face Amount $250,000.00, Thomas P. Whitfield (Primary, 50%, $125,000.00), Emily J. Whitfield (Primary, 50%, $125,000.00), without contingent beneficiary James R. Whitfield (mention of his exclusion is acceptable).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n issue_key_upper = issue_key.upper()\n # Top-level comments dict\n for ck, cv in data.get(\"comments\", {}).items():\n if ck.upper() == issue_key_upper:\n if isinstance(cv, list):\n for comment in cv:\n body = comment.get(\"body\", {}) if isinstance(comment, dict) else comment\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n # Inline in issue fields\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n issue = None\n if isinstance(issues, dict):\n for k, v in issues.items():\n if k.upper() == issue_key_upper:\n issue = v\n break\n if isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == issue_key_upper:\n issue = item\n break\n if issue and isinstance(issue, dict):\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {}) if isinstance(c, dict) else c\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef find_issue(data, issue_key):\n \"\"\"Find an issue by key (case-insensitive) in various data shapes.\"\"\"\n issue_key_upper = issue_key.upper()\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n if isinstance(issues, dict):\n for key, val in issues.items():\n if key.upper() == issue_key_upper:\n return val\n if isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == issue_key_upper:\n return item\n if isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == issue_key_upper:\n return item\n return None\n\n\ndef get_status_name(issue):\n if not isinstance(issue, dict):\n return \"\"\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n return str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n return sf.strip()\n ts = issue.get(\"status\")\n if isinstance(ts, dict):\n return str(ts.get(\"name\", \"\")).strip()\n elif isinstance(ts, str):\n return ts.strip()\n return \"\"\n\n\ndef check_routed(issue, status_lower):\n \"\"\"Check if the issue has been transitioned to 'Route to AP for Payout'.\"\"\"\n if \"route to ap for payout\" in status_lower:\n return True\n # Check transitions history\n transitions = issue.get(\"transitions\", [])\n if isinstance(transitions, list):\n for t in transitions:\n t_name = \"\"\n if isinstance(t, dict):\n t_name = str(t.get(\"name\", \"\")).lower()\n to_field = t.get(\"to\", {})\n if isinstance(to_field, dict):\n t_name += \" \" + str(to_field.get(\"name\", \"\")).lower()\n elif isinstance(to_field, str):\n t_name += \" \" + to_field.lower()\n elif isinstance(t, str):\n t_name = t.lower()\n if \"route to ap for payout\" in t_name:\n return True\n # Check changelog\n changelog = issue.get(\"changelog\", {})\n if isinstance(changelog, dict):\n histories = changelog.get(\"histories\", [])\n if isinstance(histories, list):\n for h in histories:\n for item in (h.get(\"items\", []) if isinstance(h, dict) else []):\n if isinstance(item, dict) and str(item.get(\"field\", \"\")).lower() == \"status\":\n Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "3c204513-4ad4-418b-b9a7-a5efa2ced421",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `jira_state.json`, CLM_LIFE-3 must NOT have CLM-910 or CLM-920 set, and must have a structured beneficiary allocation comment with pro-rata redistribution: Mark H. Drummond (63.64% [allow 63.6%], $381,818.18) and Susan E. Drummond (36.36% [allow 36.3%], $218,181.82), excluding deceased Linda M. Drummond and contingent George T. Drummond. Linda and George mentioned as excluded or not receiving payout is acceptable.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\n\n Block-level nodes (paragraph, heading, list item, etc.) are separated with\n newlines, and hardBreak nodes emit a newline, so the per-line payout checks\n below see the same line boundaries the comment author intended (ADF\n otherwise drops them, collapsing the whole comment onto one line).\n \"\"\"\n block_types = {\"paragraph\", \"heading\", \"blockquote\", \"listItem\",\n \"bulletList\", \"orderedList\", \"codeBlock\", \"rule\", \"tableRow\"}\n text = \"\"\n if isinstance(node, dict):\n node_type = node.get(\"type\")\n if node_type == \"text\":\n text += node.get(\"text\", \"\")\n elif node_type == \"hardBreak\":\n text += \"\\n\"\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n if node_type in block_types:\n text += \"\\n\"\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n # Top-level comments keyed by issue\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n # Inline comments inside issues -> fields -> comment\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n # --- Locate CLM_LIFE-3 ---\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n issue = None\n if isinstance(issues, dict):\n issue = issues.get(\"CLM_LIFE-3\")\n if issue is None:\n for key, val in issues.items():\n if key.upper() == \"CLM_LIFE-3\":\n issue = val\n break\n if issue is None and isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == \"CLM_LIFE-3\":\n issue = item\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-3 not found in jira_state.json\"}\n\n # --- Extract status name ---\n def get_status_name(iss):\n fields = iss.get(\"fields\", {})\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n return str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n return sf.strip()\n ts = iss.get(\"status\")\n if isinstance(ts, dict):\n return str(ts.get(\"name\", \"\")).strip()\n elif isinstance(ts, str):\n return ts.strip()\n return \"\"\n\n status_name = get_status_name(issue)\n status_lower = status_name.lower()\n\n # --- Extract labels ---\n labels = []\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n raw_labels = fields.get(\"labels\", [])\n if isinstance(raw_labels, list):\n labels = [str(l).lower() for l in raw_labels]\n\n # --- Check CLM-910 NOT set (status and labels) ---\n has_910 = (\n \"clm-910\" in status_lower\n or \"identity verification hold\" in status_lower\n or any(\"clm-910\" in lbl for lbl in labels)\n or any(\"identity-verification-hold\" in lbl or \"identity verification hold\" in lbl for lbl in labels)\n )\n\n # --- Check CLM-920 NOT set (status and labels) ---\n has_920 = (\n \"clm-920\" in status_lower\n or \"complex beneficiary hold\" in status_lower\n or anyLine truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "d2411716-ed63-4513-8756-51f430561e46",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `jira_state.json`, CLM_LIFE-4 must NOT have CLM-910 or CLM-920 set, and must have a structured beneficiary allocation comment containing Policy LC-2020-11467, Face Amount $350,000.00, Emeka A. Okafor (Primary, 70%, $245,000.00), Chidi E. Okafor (Primary, 30%, $105,000.00), without contingent beneficiary Ngozi F. Okafor (mention of her exclusion is acceptable).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n # Top-level comments dict\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n # Inline comments in issue fields\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {}) if isinstance(issue, dict) else {}\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n # --- Locate CLM_LIFE-4 ---\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n issue = None\n if isinstance(issues, dict):\n issue = issues.get(\"CLM_LIFE-4\")\n if issue is None:\n for key, val in issues.items():\n if key.upper() == \"CLM_LIFE-4\":\n issue = val\n break\n if issue is None and isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == \"CLM_LIFE-4\":\n issue = item\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-4 not found in jira_state.json\"}\n\n # --- Extract status name ---\n def get_status_name(iss):\n fields = iss.get(\"fields\", {})\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n return str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n return sf.strip()\n ts = iss.get(\"status\")\n if isinstance(ts, dict):\n return str(ts.get(\"name\", \"\")).strip()\n elif isinstance(ts, str):\n return ts.strip()\n return \"\"\n\n status_name = get_status_name(issue)\n status_lower = status_name.lower()\n\n # --- Extract labels ---\n labels = []\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n raw_labels = fields.get(\"labels\", [])\n if isinstance(raw_labels, list):\n labels = [str(l).lower() for l in raw_labels]\n\n # --- Collect all text fields from the issue for hold detection ---\n def all_field_values_str(iss):\n \"\"\"Gather string representations of all field values for broad searching.\"\"\"\n parts = []\n flds = iss.get(\"fields\", {})\n if isinstance(flds, dict):\n for k, v in flds.items():\n if k == \"comment\":\n continue # skip comments for hold detection\n if isinstance(v, str):\n parts.append(v.lower())\n elif isinstance(v, dict):\n parts.append(json.dumps(v).lower())\n elif isinstance(v, list):\n parts.append(json.dumps(v).lower())\n return \" \".join(parts)\n\n all_fields_text = all_field_values_str(issue)\n\n # --- Check CLM-910 NOT set ---\n has_910 = (\n \"clm-910\" in status_lower\n or \"identity verification hold\" in status_lower\n or any(\"clm-910\" in lbl for lbl in labels)\n or any(\"identity-verification-hold\" in lbl or \"identity verification hold\" in lbl for lbl in labels)\n or \"clm-910\" in all_fields_text\n Line truncated
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775973366733",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "CLM_LIFE-1 must NOT be transitioned to CLM_LEGAL or routed to Claims Supervisor (CLM-Identity).",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n failures = []\n\n # --- Check 1: Were any new CLM_LEGAL issues created that reference CLM_LIFE-1? ---\n # CLM_LEGAL-1 is pre-existing (Queue Roles & Assignees), so skip it\n for key, val in issues.items():\n if \"legal\" in key.lower() and key != \"CLM_LEGAL-1\":\n issue_str = json.dumps(val)\n if \"CLM_LIFE-1\" in issue_str or \"Martinez\" in issue_str or \"LC-2019-08844\" in issue_str:\n summary = \"\"\n if isinstance(val, dict):\n fields = val.get(\"fields\", {})\n if isinstance(fields, dict):\n summary = str(fields.get(\"summary\", \"\"))\n failures.append(f\"New CLM_LEGAL issue '{key}' was created referencing CLM_LIFE-1 (summary: '{summary}')\")\n\n # --- Check 2: Did the model's comments on CLM_LIFE-1 indicate routing to Legal? ---\n comments_text = get_all_comment_text(data, \"CLM_LIFE-1\")\n ct_lower = comments_text.lower()\n\n routed_to_legal = False\n if \"clm-legal\" in ct_lower:\n legal_patterns = [\n r\"rout\\w*\\s+to\\s+clm[- ]?legal\",\n r\"set\\s+clm[- ]?legal\",\n r\"created?\\s+.*clm[- ]?legal\",\n r\"assign\\w*\\s+.*clm[- ]?legal\",\n r\"forward\\w*\\s+.*clm[- ]?legal\",\n r\"escala\\w*\\s+.*clm[- ]?legal\",\n ]\n for pat in legal_patterns:\n if re.search(pat, ct_lower):\n routed_to_legal = True\n break\n\n if routed_to_legal:\n failures.append(\"CLM_LIFE-1 comments indicate routing to CLM_LEGAL\")\n\n # --- Check 3: Is the status itself set to a Legal queue? ---\n issue = issues.get(\"CLM_LIFE-1\")\n if issue is not None:\n fields = issue.get(\"fields\", {})\n status_name = \"\"\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n status_name = str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n status_name = sf.strip()\n\n if \"legal\" in status_name.lower():\n failures.append(f\"CLM_LIFE-1 status is set to a Legal queue ('{status_name}')\")\n\n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-1 incorrectly routed to Legal: \" + \"; \".join(failures)}\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"CLM_LIFE-1: Not routed to CLM_LEGAL, no Legal issues created for this claim\",\n }",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "rubric_1775973378903",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "CLM_LIFE-4 must NOT be transitioned to CLM_LEGAL or have any hold status applied.",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n failures = []\n\n # --- Check 1: Were any new CLM_LEGAL issues created that reference CLM_LIFE-4? ---\n for key, val in issues.items():\n if \"legal\" in key.lower() and key != \"CLM_LEGAL-1\":\n issue_str = json.dumps(val)\n if \"CLM_LIFE-4\" in issue_str or \"Okafor\" in issue_str or \"LC-2020-11467\" in issue_str:\n summary = \"\"\n if isinstance(val, dict):\n fields = val.get(\"fields\", {})\n if isinstance(fields, dict):\n summary = str(fields.get(\"summary\", \"\"))\n failures.append(f\"New CLM_LEGAL issue '{key}' was created referencing CLM_LIFE-4 (summary: '{summary}')\")\n\n # --- Check 2: Did the model's comments on CLM_LIFE-4 indicate routing to Legal? ---\n comments_text = get_all_comment_text(data, \"CLM_LIFE-4\")\n ct_lower = comments_text.lower()\n\n routed_to_legal = False\n if \"clm-legal\" in ct_lower:\n legal_patterns = [\n r\"rout\\w*\\s+to\\s+clm[- ]?legal\",\n r\"set\\s+clm[- ]?legal\",\n r\"created?\\s+.*clm[- ]?legal\",\n r\"assign\\w*\\s+.*clm[- ]?legal\",\n r\"forward\\w*\\s+.*clm[- ]?legal\",\n r\"escala\\w*\\s+.*clm[- ]?legal\",\n ]\n for pat in legal_patterns:\n if re.search(pat, ct_lower):\n routed_to_legal = True\n break\n\n if routed_to_legal:\n failures.append(\"CLM_LIFE-4 comments indicate routing to CLM_LEGAL\")\n\n # --- Check 3: Is the status itself set to a Legal queue? ---\n issue = issues.get(\"CLM_LIFE-4\")\n if issue is not None:\n fields = issue.get(\"fields\", {})\n status_name = \"\"\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n status_name = str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n status_name = sf.strip()\n\n if \"legal\" in status_name.lower():\n failures.append(f\"CLM_LIFE-4 status is set to a Legal queue ('{status_name}')\")\n\n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-4 incorrectly routed to Legal: \" + \"; \".join(failures)}\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"CLM_LIFE-4: Not routed to CLM_LEGAL, no Legal issues created for this claim\",\n }",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run bundled SOP Python verifiers inside a Harbor task container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
WORKDIR = Path("/workdir")
|
||||
DATA_DIR = Path("/data")
|
||||
INITIAL_DATA_DIR = Path("/initial_data")
|
||||
TESTS_DIR = Path("/tests")
|
||||
VERIFIER_DIR = Path("/logs/verifier")
|
||||
|
||||
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
|
||||
"slack": ("slack.json", ("slack.json", "slack_data.json")),
|
||||
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
|
||||
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
|
||||
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
|
||||
"shopify": ("shopify_data.json", ("shopify_data.json",)),
|
||||
}
|
||||
|
||||
|
||||
def _state_path(service: str, seed_name: str) -> Path | None:
|
||||
candidates = [
|
||||
DATA_DIR / service / "final.json",
|
||||
DATA_DIR / service / seed_name,
|
||||
INITIAL_DATA_DIR / service / seed_name,
|
||||
]
|
||||
return next((p for p in candidates if p.is_file()), None)
|
||||
|
||||
|
||||
def _build_compat_external_services(dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
|
||||
src = _state_path(service, seed_name)
|
||||
if src is not None:
|
||||
for compat_name in compat_names:
|
||||
shutil.copy2(src, dest / compat_name)
|
||||
|
||||
|
||||
def _coerce_result(raw: Any) -> dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
passed = bool(raw.get("pass", raw.get("passed", False)))
|
||||
score = raw.get("score", 1.0 if passed else 0.0)
|
||||
try:
|
||||
score = float(score)
|
||||
except (TypeError, ValueError):
|
||||
score = 1.0 if passed else 0.0
|
||||
return {
|
||||
"pass": passed,
|
||||
"score": max(0.0, min(1.0, score)),
|
||||
"feedback": str(raw.get("feedback", "")),
|
||||
}
|
||||
passed = bool(raw)
|
||||
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
|
||||
|
||||
|
||||
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
|
||||
rubric_id = str(rubric.get("id") or "rubric")
|
||||
code = rubric.get("verifier_code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": "rubric has no verifier_code",
|
||||
}
|
||||
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
|
||||
verify = namespace.get("verify")
|
||||
if not callable(verify):
|
||||
raise RuntimeError("verifier_code did not define verify()")
|
||||
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
|
||||
return {"id": rubric_id, **result}
|
||||
except Exception:
|
||||
return {
|
||||
"id": rubric_id,
|
||||
"pass": False,
|
||||
"score": 0.0,
|
||||
"feedback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rubrics_path = TESTS_DIR / "rubrics.json"
|
||||
if not rubrics_path.is_file():
|
||||
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
rubrics = json.loads(rubrics_path.read_text())
|
||||
if not isinstance(rubrics, list):
|
||||
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
|
||||
compat_dir = Path(tmp)
|
||||
_build_compat_external_services(compat_dir)
|
||||
results = [_run_one(r, compat_dir) for r in rubrics]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.get("pass"))
|
||||
average_score = round(
|
||||
sum(float(r.get("score", 0.0)) for r in results) / total,
|
||||
4,
|
||||
) if total else 0.0
|
||||
|
||||
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
|
||||
for result in results:
|
||||
status = "PASS" if result.get("pass") else "FAIL"
|
||||
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
|
||||
print(f" [{status}] {result.get('id')}: {feedback}")
|
||||
|
||||
output = {
|
||||
"passed": passed == total,
|
||||
"rubrics_passed": passed,
|
||||
"rubrics_total": total,
|
||||
"score": average_score,
|
||||
"rubric_results": results,
|
||||
}
|
||||
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
|
||||
|
||||
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
Today is April 10, 2026, and there are several multi-beneficiary life claims that need to be processed. Can you find these in Jira and process per the Vanguard_Shield_Core_Operations_SOP and complete all required actions.
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
pip install openpyxl pdfplumber python-docx 2>/dev/null
|
||||
python /tests/sop_verifier.py
|
||||
Reference in new issue
Block a user