52 lines
54 KiB
JSON
52 lines
54 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "fd5fea82-8bfe-48e1-81d0-be8b87856e35",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In `jira_state.json`, CLM_LIFE-1 must NOT have CLM-910 (Identity Verification Hold) or CLM-920 (Complex Beneficiary Hold) set.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n # --- Locate CLM_LIFE-1 ---\n issue = None\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n\n if isinstance(issues, dict):\n issue = issues.get(\"CLM_LIFE-1\")\n if issue is None:\n for key, val in issues.items():\n if key.upper() == \"CLM_LIFE-1\":\n issue = val\n break\n\n if issue is None and isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == \"CLM_LIFE-1\":\n issue = item\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-1 not found in jira_state.json\"}\n\n # --- Extract status name (handles nested dict or plain string) ---\n def get_status_name(issue):\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n status_field = fields.get(\"status\")\n if isinstance(status_field, dict):\n return str(status_field.get(\"name\", \"\")).strip()\n elif isinstance(status_field, str):\n return status_field.strip()\n top_status = issue.get(\"status\")\n if isinstance(top_status, dict):\n return str(top_status.get(\"name\", \"\")).strip()\n elif isinstance(top_status, str):\n return top_status.strip()\n return \"\"\n\n status_name = get_status_name(issue)\n status_lower = status_name.lower()\n\n # --- Extract labels ---\n labels = []\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n raw_labels = fields.get(\"labels\", [])\n if isinstance(raw_labels, list):\n labels = [str(l).lower() for l in raw_labels]\n\n # --- Check CLM-910 NOT set (status and labels only, NOT comment text) ---\n has_910 = (\n \"clm-910\" in status_lower\n or \"identity verification hold\" in status_lower\n or any(\"clm-910\" in lbl for lbl in labels)\n or any(\"identity-verification-hold\" in lbl or \"identity verification hold\" in lbl for lbl in labels)\n )\n\n # --- Check CLM-920 NOT set (status and labels only) ---\n has_920 = (\n \"clm-920\" in status_lower\n or \"complex beneficiary hold\" in status_lower\n or any(\"clm-920\" in lbl for lbl in labels)\n or any(\"complex-beneficiary-hold\" in lbl or \"complex beneficiary hold\" in lbl for lbl in labels)\n )\n\n # --- Check status is \"Route to AP for Payout\" ---\n routed = \"route to ap for payout\" in status_lower\n\n if not routed:\n transitions = issue.get(\"transitions\", [])\n if isinstance(transitions, list):\n for t in transitions:\n t_name = \"\"\n if isinstance(t, dict):\n t_name = str(t.get(\"name\", t.get(\"to\", \"\"))).lower()\n elif isinstance(t, str):\n t_name = t.lower()\n if \"route to ap for payout\" in t_name:\n routed = True\n break\n\n # --- Build result ---\n issues_found = []\n if has_910:\n issues_found.append(f\"CLM-910 Identity Verification Hold is set (status: '{status_name}')\")\n if has_920:\n issues_found.append(f\"CLM-920 Complex Beneficiary Hold is set (status: '{status_name}')\")\n if issues_found:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-1:
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "3328f333-48c7-45b0-a940-a7e9e52f2b5f",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In `jira_state.json`, CLM_LIFE-1 must have a structured beneficiary allocation comment containing Policy LC-2019-08844, Face Amount $400,000.00, Maria L. Martinez (Primary, 60%, $240,000.00), David R. Martinez (Primary, 40%, $160,000.00), with a header line and Analyst/Date attribution (\"analyst\" and the date must appear within 80 characters of each other), and must NOT include Elena Martinez (mention of her exclusion is acceptable).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n # 1. Top-level comments dict\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n # 2. Inline comments on the issue (fields.comment.comments)\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {}) if isinstance(issue, dict) else {}\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n # --- Locate CLM_LIFE-1 ---\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n issue = None\n if isinstance(issues, dict):\n issue = issues.get(\"CLM_LIFE-1\")\n if issue is None:\n for key, val in issues.items():\n if key.upper() == \"CLM_LIFE-1\":\n issue = val\n break\n if issue is None and isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == \"CLM_LIFE-1\":\n issue = item\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-1 not found in jira_state.json\"}\n\n # --- Get all comment text (ADF-aware, checks both top-level and inline) ---\n comments_text = get_all_comment_text(data, \"CLM_LIFE-1\")\n ct_lower = comments_text.lower()\n\n failures = []\n\n # Check header line: \"beneficiary allocation\" somewhere in the comment\n if \"beneficiary allocation\" not in ct_lower:\n failures.append(\"Missing 'Beneficiary Allocation' header line\")\n\n # Check policy LC-2019-08844\n if \"lc-2019-08844\" not in ct_lower:\n failures.append(\"Missing Policy LC-2019-08844\")\n\n # Check face amount $400,000.00\n if \"400,000\" not in comments_text and \"400000\" not in comments_text:\n failures.append(\"Missing Face Amount $400,000.00\")\n\n # Check Maria L. Martinez (Primary, 60%, $240,000.00)\n if \"maria\" not in ct_lower or \"martinez\" not in ct_lower:\n failures.append(\"Missing beneficiary Maria L. Martinez\")\n else:\n if \"240,000\" not in comments_text and \"240000\" not in comments_text:\n failures.append(\"Missing Maria Martinez amount $240,000.00\")\n if \"60%\" not in comments_text and \"60 %\" not in comments_text:\n failures.append(\"Missing Maria Martinez 60% split\")\n\n # Check David R. Martinez (Primary, 40%, $160,000.00)\n if \"david\" not in ct_lower:\n failures.append(\"Missing beneficia
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "819c155d-7a13-4695-978e-f567c5be3cc9",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In `jira_state.json`, CLM_LIFE-2 must NOT have CLM-910 or CLM-920 set, and must have a structured beneficiary allocation comment containing Policy LC-2021-15290, Face Amount $250,000.00, Thomas P. Whitfield (Primary, 50%, $125,000.00), Emily J. Whitfield (Primary, 50%, $125,000.00), without contingent beneficiary James R. Whitfield (mention of his exclusion is acceptable).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n issue_key_upper = issue_key.upper()\n # Top-level comments dict\n for ck, cv in data.get(\"comments\", {}).items():\n if ck.upper() == issue_key_upper:\n if isinstance(cv, list):\n for comment in cv:\n body = comment.get(\"body\", {}) if isinstance(comment, dict) else comment\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n # Inline in issue fields\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n issue = None\n if isinstance(issues, dict):\n for k, v in issues.items():\n if k.upper() == issue_key_upper:\n issue = v\n break\n if isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == issue_key_upper:\n issue = item\n break\n if issue and isinstance(issue, dict):\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {}) if isinstance(c, dict) else c\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef find_issue(data, issue_key):\n \"\"\"Find an issue by key (case-insensitive) in various data shapes.\"\"\"\n issue_key_upper = issue_key.upper()\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n if isinstance(issues, dict):\n for key, val in issues.items():\n if key.upper() == issue_key_upper:\n return val\n if isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == issue_key_upper:\n return item\n if isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == issue_key_upper:\n return item\n return None\n\n\ndef get_status_name(issue):\n if not isinstance(issue, dict):\n return \"\"\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n return str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n return sf.strip()\n ts = issue.get(\"status\")\n if isinstance(ts, dict):\n return str(ts.get(\"name\", \"\")).strip()\n elif isinstance(ts, str):\n return ts.strip()\n return \"\"\n\n\ndef check_routed(issue, status_lower):\n \"\"\"Check if the issue has been transitioned to 'Route to AP for Payout'.\"\"\"\n if \"route to ap for payout\" in status_lower:\n return True\n # Check transitions history\n transitions = issue.get(\"transitions\", [])\n if isinstance(transitions, list):\n for t in transitions:\n t_name = \"\"\n if isinstance(t, dict):\n t_name = str(
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "3c204513-4ad4-418b-b9a7-a5efa2ced421",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In `jira_state.json`, CLM_LIFE-3 must NOT have CLM-910 or CLM-920 set, and must have a structured beneficiary allocation comment with pro-rata redistribution: Mark H. Drummond (63.64% [allow 63.6%], $381,818.18) and Susan E. Drummond (36.36% [allow 36.3%], $218,181.82), excluding deceased Linda M. Drummond and contingent George T. Drummond. Linda and George mentioned as excluded or not receiving payout is acceptable.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\n\n Block-level nodes (paragraph, heading, list item, etc.) are separated with\n newlines, and hardBreak nodes emit a newline, so the per-line payout checks\n below see the same line boundaries the comment author intended (ADF\n otherwise drops them, collapsing the whole comment onto one line).\n \"\"\"\n block_types = {\"paragraph\", \"heading\", \"blockquote\", \"listItem\",\n \"bulletList\", \"orderedList\", \"codeBlock\", \"rule\", \"tableRow\"}\n text = \"\"\n if isinstance(node, dict):\n node_type = node.get(\"type\")\n if node_type == \"text\":\n text += node.get(\"text\", \"\")\n elif node_type == \"hardBreak\":\n text += \"\\n\"\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n if node_type in block_types:\n text += \"\\n\"\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n # Top-level comments keyed by issue\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n # Inline comments inside issues -> fields -> comment\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n # --- Locate CLM_LIFE-3 ---\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n issue = None\n if isinstance(issues, dict):\n issue = issues.get(\"CLM_LIFE-3\")\n if issue is None:\n for key, val in issues.items():\n if key.upper() == \"CLM_LIFE-3\":\n issue = val\n break\n if issue is None and isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == \"CLM_LIFE-3\":\n issue = item\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-3 not found in jira_state.json\"}\n\n # --- Extract status name ---\n def get_status_name(iss):\n fields = iss.get(\"fields\", {})\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n return str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n return sf.strip()\n ts = iss.get(\"status\")\n if isinstance(ts, dict):\n return str(ts.get(\"name\", \"\")).strip()\n elif isinstance(ts, str):\n return ts.strip()\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "d2411716-ed63-4513-8756-51f430561e46",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In `jira_state.json`, CLM_LIFE-4 must NOT have CLM-910 or CLM-920 set, and must have a structured beneficiary allocation comment containing Policy LC-2020-11467, Face Amount $350,000.00, Emeka A. Okafor (Primary, 70%, $245,000.00), Chidi E. Okafor (Primary, 30%, $105,000.00), without contingent beneficiary Ngozi F. Okafor (mention of her exclusion is acceptable).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n # Top-level comments dict\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n # Inline comments in issue fields\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {}) if isinstance(issue, dict) else {}\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n if isinstance(body, str):\n text += body + \"\\n\"\n else:\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n # --- Locate CLM_LIFE-4 ---\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n issue = None\n if isinstance(issues, dict):\n issue = issues.get(\"CLM_LIFE-4\")\n if issue is None:\n for key, val in issues.items():\n if key.upper() == \"CLM_LIFE-4\":\n issue = val\n break\n if issue is None and isinstance(data, list):\n for item in data:\n if isinstance(item, dict) and item.get(\"key\", \"\").upper() == \"CLM_LIFE-4\":\n issue = item\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"CLM_LIFE-4 not found in jira_state.json\"}\n\n # --- Extract status name ---\n def get_status_name(iss):\n fields = iss.get(\"fields\", {})\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n return str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n return sf.strip()\n ts = iss.get(\"status\")\n if isinstance(ts, dict):\n return str(ts.get(\"name\", \"\")).strip()\n elif isinstance(ts, str):\n return ts.strip()\n return \"\"\n\n status_name = get_status_name(issue)\n status_lower = status_name.lower()\n\n # --- Extract labels ---\n labels = []\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n raw_labels = fields.get(\"labels\", [])\n if isinstance(raw_labels, list):\n labels = [str(l).lower() for l in raw_labels]\n\n # --- Collect all text fields from the issue for hold detection ---\n def all_field_values_str(iss):\n \"\"\"Gather string representations of all field values for broad searching.\"\"\"\n parts = []\n flds = iss.get(\"fields\", {})\n if
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775973366733",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "CLM_LIFE-1 must NOT be transitioned to CLM_LEGAL or routed to Claims Supervisor (CLM-Identity).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n failures = []\n\n # --- Check 1: Were any new CLM_LEGAL issues created that reference CLM_LIFE-1? ---\n # CLM_LEGAL-1 is pre-existing (Queue Roles & Assignees), so skip it\n for key, val in issues.items():\n if \"legal\" in key.lower() and key != \"CLM_LEGAL-1\":\n issue_str = json.dumps(val)\n if \"CLM_LIFE-1\" in issue_str or \"Martinez\" in issue_str or \"LC-2019-08844\" in issue_str:\n summary = \"\"\n if isinstance(val, dict):\n fields = val.get(\"fields\", {})\n if isinstance(fields, dict):\n summary = str(fields.get(\"summary\", \"\"))\n failures.append(f\"New CLM_LEGAL issue '{key}' was created referencing CLM_LIFE-1 (summary: '{summary}')\")\n\n # --- Check 2: Did the model's comments on CLM_LIFE-1 indicate routing to Legal? ---\n comments_text = get_all_comment_text(data, \"CLM_LIFE-1\")\n ct_lower = comments_text.lower()\n\n routed_to_legal = False\n if \"clm-legal\" in ct_lower:\n legal_patterns = [\n r\"rout\\w*\\s+to\\s+clm[- ]?legal\",\n r\"set\\s+clm[- ]?legal\",\n r\"created?\\s+.*clm[- ]?legal\",\n r\"assign\\w*\\s+.*clm[- ]?legal\",\n r\"forward\\w*\\s+.*clm[- ]?legal\",\n r\"escala\\w*\\s+.*clm[- ]?legal\",\n ]\n for pat in legal_patterns:\n if re.search(pat, ct_lower):\n routed_to_legal = True\n break\n\n if routed_to_legal:\n failures.append(\"CLM_LIFE-1 comments indicate routing to CLM_LEGAL\")\n\n # --- Check 3: Is the status itself set to a Legal queue? ---\n issue = issues.get(\"CLM_LIFE-1\")\n if issue is not None:\n fields = issue.get(\"fields\", {})\n status_name = \"\"\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n status_name = str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n status_name = sf.strip()\n\n if \"legal\" in status_name.lower():\n failures
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775973378903",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "CLM_LIFE-4 must NOT be transitioned to CLM_LEGAL or have any hold status applied.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef extract_text_from_adf(node):\n \"\"\"Recursively extract plain text from Atlassian Document Format.\"\"\"\n text = \"\"\n if isinstance(node, dict):\n if node.get(\"type\") == \"text\":\n text += node.get(\"text\", \"\")\n for child in node.get(\"content\", []):\n text += extract_text_from_adf(child)\n elif isinstance(node, list):\n for item in node:\n text += extract_text_from_adf(item)\n return text\n\n\ndef get_all_comment_text(data, issue_key):\n \"\"\"Get all comment text for an issue from both top-level and inline locations.\"\"\"\n text = \"\"\n for comment in data.get(\"comments\", {}).get(issue_key, []):\n body = comment.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key, {}) if isinstance(issues, dict) else {}\n fields = issue.get(\"fields\", {})\n if isinstance(fields, dict):\n comment_field = fields.get(\"comment\", {})\n if isinstance(comment_field, dict):\n for c in comment_field.get(\"comments\", []):\n body = c.get(\"body\", {})\n text += extract_text_from_adf(body) + \"\\n\"\n return text\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n\n with open(jira_path) as f:\n data = json.load(f)\n\n issues = data.get(\"issues\", {}) if isinstance(data, dict) else {}\n failures = []\n\n # --- Check 1: Were any new CLM_LEGAL issues created that reference CLM_LIFE-4? ---\n for key, val in issues.items():\n if \"legal\" in key.lower() and key != \"CLM_LEGAL-1\":\n issue_str = json.dumps(val)\n if \"CLM_LIFE-4\" in issue_str or \"Okafor\" in issue_str or \"LC-2020-11467\" in issue_str:\n summary = \"\"\n if isinstance(val, dict):\n fields = val.get(\"fields\", {})\n if isinstance(fields, dict):\n summary = str(fields.get(\"summary\", \"\"))\n failures.append(f\"New CLM_LEGAL issue '{key}' was created referencing CLM_LIFE-4 (summary: '{summary}')\")\n\n # --- Check 2: Did the model's comments on CLM_LIFE-4 indicate routing to Legal? ---\n comments_text = get_all_comment_text(data, \"CLM_LIFE-4\")\n ct_lower = comments_text.lower()\n\n routed_to_legal = False\n if \"clm-legal\" in ct_lower:\n legal_patterns = [\n r\"rout\\w*\\s+to\\s+clm[- ]?legal\",\n r\"set\\s+clm[- ]?legal\",\n r\"created?\\s+.*clm[- ]?legal\",\n r\"assign\\w*\\s+.*clm[- ]?legal\",\n r\"forward\\w*\\s+.*clm[- ]?legal\",\n r\"escala\\w*\\s+.*clm[- ]?legal\",\n ]\n for pat in legal_patterns:\n if re.search(pat, ct_lower):\n routed_to_legal = True\n break\n\n if routed_to_legal:\n failures.append(\"CLM_LIFE-4 comments indicate routing to CLM_LEGAL\")\n\n # --- Check 3: Is the status itself set to a Legal queue? ---\n issue = issues.get(\"CLM_LIFE-4\")\n if issue is not None:\n fields = issue.get(\"fields\", {})\n status_name = \"\"\n if isinstance(fields, dict):\n sf = fields.get(\"status\")\n if isinstance(sf, dict):\n status_name = str(sf.get(\"name\", \"\")).strip()\n elif isinstance(sf, str):\n status_name = sf.strip()\n\n if \"legal\" in status_name.lower():\n failures.append(f\"CLM_LIFE-4 status is set to a Legal queue ('{status_name}')\")\n
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
}
|
||
|
|
]
|