80 lines
62 KiB
JSON
80 lines
62 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "eccaf388-6d88-479e-ae0a-734020b8b053",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In external_data/final/jira_state.json, issue OPS-3 must have at least one comment (stored either in the issue's own comment field or in the top-level comments collection) where the text references VSM Branded Water Bottles, water bottle, or Oct 1, AND flags the expense as missing business justification/unauthorized vendor.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Try multiple paths to find jira_state.json\n paths_to_try = []\n if external_services_path:\n paths_to_try.append(Path(external_services_path) / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'external_data' / 'final' / 'jira_state.json')\n\n jira_data = None\n found_path = None\n for p in paths_to_try:\n if p.exists():\n with open(p) as f:\n jira_data = json.load(f)\n found_path = str(p)\n break\n\n if jira_data is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found in any expected location'}\n\n issue_id = 'OPS-3'\n issue = None\n\n if isinstance(jira_data, dict):\n if 'issues' in jira_data:\n issues_obj = jira_data['issues']\n if isinstance(issues_obj, dict):\n if issue_id in issues_obj:\n issue = issues_obj[issue_id]\n else:\n for key, val in issues_obj.items():\n if isinstance(val, dict) and (val.get('id') == issue_id or val.get('key') == issue_id):\n issue = val\n break\n elif isinstance(issues_obj, list):\n for iss in issues_obj:\n if isinstance(iss, dict) and (iss.get('id') == issue_id or iss.get('key') == issue_id):\n issue = iss\n break\n\n if issue is None and issue_id in jira_data:\n issue = jira_data[issue_id]\n\n if issue is None:\n for key, val in jira_data.items():\n if isinstance(val, dict):\n if val.get('id') == issue_id or val.get('key') == issue_id:\n issue = val\n break\n elif isinstance(jira_data, list):\n for iss in jira_data:\n if isinstance(iss, dict) and (iss.get('id') == issue_id or iss.get('key') == issue_id):\n issue = iss\n break\n\n # Collect comments from BOTH the issue object AND the top-level comments dict\n comments = []\n\n # 1) Comments stored on the issue itself\n if issue is not None:\n issue_comments = issue.get('comments', [])\n if isinstance(issue_comments, list):\n comments.extend(issue_comments)\n fields = issue.get('fields', {})\n if isinstance(fields, dict):\n comment_field = fields.get('comment', {})\n if isinstance(comment_field, dict):\n inner = comment_field.get('comments', [])\n if isinstance(inner, list):\n comments.extend(inner)\n elif isinstance(comment_field, list):\n comments.extend(comment_field)\n\n # 2) Comments stored in the top-level 'comments' collection, keyed by issue id\n if isinstance(jira_data, dict):\n top_comments = jira_data.get('comments')\n if isinstance(top_comments, dict):\n entry = top_comments.get(issue_id)\n if isinstance(entry, list):\n comments.extend(entry)\n elif isinstance(entry, dict):\n # in case it's wrapped, e.g. {'comments': [...]}\n inner = entry.get('comments')\n if isinstance(inner, list):\n comments.extend(inner)\n\n if issue is None and not comments:\n return {'pass': False, 'score': 0.0, 'feedback': f'Issue {issue_id} not found and no top-level comments for it either (loaded from {found_path}).'}\n\n def extract_adf_text(node):\n \"\"\"Recursively pull every 'text' string out of an Atlassian Document Format dict.\"\"\"\n out = []\n if isinstance(node, dict):\n if isinstance(node.get('text'), str):\n out.append(node['text'])\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "c713cced-31be-4221-8419-072a071ecab7",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In jira_state.json, issue OPS-3 must NOT contain any comment where\nthe text contains ('Capital Grille' or 'Marriott' or 'Hartwell Group')\nAND contains 'Missing business justification'",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n paths_to_try = []\n if external_services_path:\n paths_to_try.append(Path(external_services_path) / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'jira_state.json')\n\n jira_data = None\n for p in paths_to_try:\n if p.exists():\n with open(p) as f:\n jira_data = json.load(f)\n break\n\n if jira_data is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'jira_state.json not found'}\n\n # Collect all issues\n all_issues = []\n if isinstance(jira_data, dict):\n if 'issues' in jira_data:\n issues_val = jira_data['issues']\n if isinstance(issues_val, list):\n all_issues = issues_val\n elif isinstance(issues_val, dict):\n all_issues = list(issues_val.values())\n else:\n all_issues = list(jira_data.values())\n elif isinstance(jira_data, list):\n all_issues = jira_data\n\n # Find issue OPS-3\n target_issue = None\n for iss in all_issues:\n if not isinstance(iss, dict):\n continue\n issue_key = str(iss.get('key', iss.get('id', iss.get('issue_key', '')))).strip()\n if issue_key.upper() == 'OPS-3':\n target_issue = iss\n break\n\n if target_issue is None:\n # Issue not found at all - that means no violating comment exists, so pass\n return {'pass': True, 'score': 1.0, 'feedback': 'Issue OPS-3 not found in jira_state.json, so no violating comment exists. PASS.'}\n\n # Get comments from the issue\n comments = []\n if 'comments' in target_issue:\n c = target_issue['comments']\n if isinstance(c, list):\n comments = c\n elif isinstance(c, dict):\n comments = list(c.values())\n # Also check nested fields structure\n if 'fields' in target_issue and isinstance(target_issue['fields'], dict):\n fields = target_issue['fields']\n if 'comment' in fields:\n comment_field = fields['comment']\n if isinstance(comment_field, dict) and 'comments' in comment_field:\n comments.extend(comment_field['comments'] if isinstance(comment_field['comments'], list) else list(comment_field['comments'].values()))\n elif isinstance(comment_field, list):\n comments.extend(comment_field)\n\n vendor_keywords = ['capital grille', 'marriott', 'hartwell group']\n justification_keyword = 'missing business justification'\n\n violating_comments = []\n for comment in comments:\n if isinstance(comment, dict):\n text = str(comment.get('body', comment.get('text', comment.get('content', '')))).lower()\n elif isinstance(comment, str):\n text = comment.lower()\n else:\n continue\n\n has_vendor = any(kw in text for kw in vendor_keywords)\n has_justification = justification_keyword in text\n\n if has_vendor and has_justification:\n violating_comments.append(text[:200])\n\n if len(violating_comments) > 0:\n return {\n 'pass': False,\n 'score': 0.0,\n 'feedback': f'FAIL: Issue OPS-3 contains {len(violating_comments)} comment(s) that mention a vendor keyword (Capital Grille/Marriott/Hartwell Group) AND \"Missing business justification\". Violating comment excerpts: {violating_comments}'\n }\n\n return {\n 'pass': True,\n 'score': 1.0,\n 'feedback': f'PASS: Issue OPS-3 found with {len(comments)} comment(s), none of which contain both a vendor keyword and \"Missing business justification\".'\n }\n",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "6a839ad9-3949-4bf7-9002-63258e6e5e43",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In external_data/final/jira_state.json, issue OPS-4 must have at least one comment (stored either in the issue's own comment field or in the top-level comments collection) where the text references Sullivan's Steakhouse or Oct 3, AND flags the expense as missing business justification/unauthorized vendor.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef _extract_adf_text(node):\n \"\"\"Recursively extract plain text from an Atlassian Document Format node.\"\"\"\n if node is None:\n return ''\n if isinstance(node, str):\n return node\n if isinstance(node, list):\n return ' '.join(_extract_adf_text(n) for n in node)\n if isinstance(node, dict):\n parts = []\n # Direct text field (ADF text nodes have type='text' and a 'text' key)\n if isinstance(node.get('text'), str):\n parts.append(node['text'])\n # Recurse into 'content' which holds child nodes\n if 'content' in node:\n parts.append(_extract_adf_text(node['content']))\n return ' '.join(p for p in parts if p)\n return ''\n\n\ndef _comment_to_text(comment):\n \"\"\"Turn a comment entry (string or dict) into a single text string.\"\"\"\n if isinstance(comment, str):\n return comment\n if not isinstance(comment, dict):\n return ''\n # Try common text-bearing fields\n for field in ['body', 'text', 'content', 'message', 'comment']:\n if field in comment:\n val = comment[field]\n if isinstance(val, str):\n return val\n if isinstance(val, dict) or isinstance(val, list):\n extracted = _extract_adf_text(val)\n if extracted:\n return extracted\n # Fallback: stringify whole comment\n return json.dumps(comment)\n\n\ndef verify(workspace_path, external_services_path=None):\n # Look for jira_state.json in multiple possible locations\n paths_to_try = []\n if external_services_path:\n paths_to_try.append(Path(external_services_path) / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'external_data' / 'final' / 'jira_state.json')\n paths_to_try.append(Path(workspace_path) / 'jira_state.json')\n\n jira_data = None\n jira_path_used = None\n for p in paths_to_try:\n if p.exists():\n with open(p) as f:\n jira_data = json.load(f)\n jira_path_used = str(p)\n break\n\n if jira_data is None:\n return {'pass': False, 'score': 0.0,\n 'feedback': 'jira_state.json not found in any expected location. Tried: '\n + str([str(p) for p in paths_to_try])}\n\n ISSUE_KEY = 'OPS-4'\n\n # ---- Collect all comments associated with the issue from any location ----\n collected_comments = []\n\n if isinstance(jira_data, dict):\n # 1) Top-level 'comments' collection keyed by issue ID (the case the old verifier missed)\n top_comments = jira_data.get('comments')\n if isinstance(top_comments, dict):\n entry = top_comments.get(ISSUE_KEY)\n if isinstance(entry, list):\n collected_comments.extend(entry)\n elif entry:\n collected_comments.append(entry)\n elif isinstance(top_comments, list):\n # Possibly a flat list with issue references on each comment\n for c in top_comments:\n if isinstance(c, dict):\n ref = c.get('issue') or c.get('issueKey') or c.get('issue_id') or c.get('key')\n if ref == ISSUE_KEY:\n collected_comments.append(c)\n\n # 2) Find the issue object itself (various shapes)\n target_issue = None\n issues_obj = jira_data.get('issues')\n if isinstance(issues_obj, dict):\n target_issue = issues_obj.get(ISSUE_KEY)\n elif isinstance(issues_obj, list):\n for issue in issues_obj:\n if isinstance(issue, dict):\n k = issue.get('key') or issue.get('id') or ''\n if k == ISSUE_KEY:\n target_issue = issue\n break\n if target_issue is None and ISSUE_KEY in jira_data:\n target_issue = jira_data[ISSUE_KEY]\n\n # 3) Pull comments embe
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775880280596",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In external_data/final/jira_state.json, issue OPS-4 must have at least one comment (stored either in the issue's own comment field or in the top-level comments collection) where the text references Hudson's Coffee or Oct 4, AND flags the expense as missing business justification/unauthorized vendor.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef extract_text_from_adf(node):\n \"\"\"Walk an Atlassian Document Format dict/list and return concatenated text.\n If given a plain string, returns it. If given anything else, returns ''.\"\"\"\n if node is None:\n return \"\"\n if isinstance(node, str):\n return node\n if isinstance(node, list):\n return \" \".join(extract_text_from_adf(n) for n in node)\n if isinstance(node, dict):\n parts = []\n # ADF text nodes have a \"text\" string field\n if isinstance(node.get(\"text\"), str):\n parts.append(node[\"text\"])\n # Recurse into \"content\" (ADF's child container)\n if \"content\" in node:\n parts.append(extract_text_from_adf(node[\"content\"]))\n return \" \".join(p for p in parts if p)\n return \"\"\n\n\ndef verify(workspace_path, external_services_path=None):\n # Look for jira_state.json in multiple locations\n jira_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"jira_state.json\")\n candidates.append(Path(external_services_path) / \"final\" / \"jira_state.json\")\n candidates.append(Path(workspace_path) / \"jira_state.json\")\n candidates.append(Path(workspace_path) / \"external_data\" / \"final\" / \"jira_state.json\")\n for base in [external_services_path, workspace_path]:\n if base:\n for p in Path(base).rglob(\"jira_state.json\"):\n candidates.append(p)\n\n for c in candidates:\n if c and c.exists():\n jira_path = c\n break\n\n if jira_path is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"Could not find jira_state.json in workspace or external services path.\"}\n\n try:\n with open(jira_path, 'r', encoding='utf-8') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse jira_state.json: {e}\"}\n\n ISSUE_KEY = \"OPS-4\"\n\n # --- Locate the issue object (may or may not exist; comments can live elsewhere) ---\n issue = None\n if isinstance(jira_data, dict):\n if \"issues\" in jira_data:\n issues = jira_data[\"issues\"]\n if isinstance(issues, dict):\n if ISSUE_KEY in issues:\n issue = issues[ISSUE_KEY]\n else:\n for v in issues.values():\n if isinstance(v, dict) and (\n v.get(\"key\") == ISSUE_KEY or v.get(\"id\") == ISSUE_KEY or v.get(\"issue_key\") == ISSUE_KEY\n ):\n issue = v\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict) and (\n item.get(\"key\") == ISSUE_KEY or item.get(\"id\") == ISSUE_KEY or item.get(\"issue_key\") == ISSUE_KEY\n ):\n issue = item\n break\n if issue is None and ISSUE_KEY in jira_data:\n issue = jira_data[ISSUE_KEY]\n elif isinstance(jira_data, list):\n for item in jira_data:\n if isinstance(item, dict) and (\n item.get(\"key\") == ISSUE_KEY or item.get(\"id\") == ISSUE_KEY or item.get(\"issue_key\") == ISSUE_KEY\n ):\n issue = item\n break\n\n # --- Collect comments from BOTH locations ---\n comments = []\n\n # Location A: top-level \"comments\" collection keyed by issue ID\n if isinstance(jira_data, dict) and isinstance(jira_data.get(\"comments\"), dict):\n top_comments = jira_data[\"comments\"].get(ISSUE_KEY)\n if isinstance(top_comments, list):\n comments.extend(top_comments)\n elif isinstance(top_comments, dict):\n comments.append(top_comm
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775880300763",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In external_data/final/jira_state.json, issue OPS-4 must NOT contain any comment where Nexus Corp or Oct 5 appears within the same sentence as Missing business justification, meaning the Nexus Corp / Oct 5 line is itself being flagged for rejection, not merely mentioned as a supported line within a broader rejection comment.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef _extract_adf_text(node):\n \"\"\"Recursively pull all 'text' fields out of an Atlassian Document Format body.\"\"\"\n parts = []\n if isinstance(node, dict):\n if isinstance(node.get(\"text\"), str):\n parts.append(node[\"text\"])\n for v in node.values():\n if isinstance(v, (dict, list)):\n parts.extend(_extract_adf_text(v))\n elif isinstance(node, list):\n for item in node:\n parts.extend(_extract_adf_text(item))\n return parts\n\ndef _comment_text(comment):\n \"\"\"Return a plain-text string for one comment entry.\"\"\"\n if isinstance(comment, str):\n return comment\n if not isinstance(comment, dict):\n return \"\"\n body = comment.get(\"body\")\n if isinstance(body, str):\n return body\n if isinstance(body, dict):\n return \" \".join(_extract_adf_text(body))\n if isinstance(body, list):\n return \" \".join(_extract_adf_text(body))\n # Fallbacks just in case\n for field in (\"text\", \"content\", \"message\"):\n val = comment.get(field)\n if isinstance(val, str):\n return val\n return \"\"\n\ndef _split_sentences(text):\n \"\"\"Split on sentence terminators and newlines. Keeps things simple.\"\"\"\n # Replace newlines with a period so list items / line breaks count as sentence boundaries.\n normalized = re.sub(r'[\\r\\n]+', '. ', text)\n # Split on . ! ? followed by space or end of string.\n sentences = re.split(r'(?<=[.!?])\\s+', normalized)\n return [s.strip() for s in sentences if s.strip()]\n\ndef verify(workspace_path, external_services_path=None):\n # Locate jira_state.json\n candidates = []\n ws = Path(workspace_path)\n candidates.append(ws / \"jira_state.json\")\n candidates.append(ws / \"external_data\" / \"final\" / \"jira_state.json\")\n if external_services_path:\n esp = Path(external_services_path)\n candidates.append(esp / \"jira_state.json\")\n candidates.append(esp / \"final\" / \"jira_state.json\")\n candidates.append(esp / \"external_data\" / \"final\" / \"jira_state.json\")\n\n jira_path = None\n for c in candidates:\n if c.exists():\n jira_path = c\n break\n if jira_path is None:\n found = list(ws.rglob(\"jira_state.json\"))\n if external_services_path:\n found += list(Path(external_services_path).rglob(\"jira_state.json\"))\n if found:\n jira_path = found[0]\n\n if jira_path is None:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"jira_state.json not found; forbidden comment cannot exist, passing by default.\"}\n\n try:\n with open(jira_path, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"Could not parse jira_state.json ({e}); passing by default.\"}\n\n # Comments live at jira_data[\"comments\"][\"OPS-4\"]\n comments = []\n if isinstance(jira_data, dict):\n top_comments = jira_data.get(\"comments\")\n if isinstance(top_comments, dict):\n entry = top_comments.get(\"OPS-4\")\n if isinstance(entry, list):\n comments = entry\n\n if not comments:\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": \"No comments found for OPS-4; forbidden comment does not exist.\"}\n\n violating = []\n for i, comment in enumerate(comments):\n text = _comment_text(comment)\n if not text:\n continue\n\n for sentence in _split_sentences(text):\n s_lower = sentence.lower()\n has_trigger = (\"nexus corp\" in s_lower) or (\"oct 5\" in s_lower) or (\"october 5\" in s_lower)\n has_missing_just = \"missing business justification\" in s_lower\n if has_trigger and has_missing_just:\n
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775880314655",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In external_data/final/jira_state.json, a standalone issue must exist under the top-level 'issues' object with 'Manager Lookup Needed' and 'T&E Exception' in its fields.summary, and 'OPS-4' in the text content of its fields.description (search nested content[].content[].text if fields.description is a document object; optionally also 'Derek Huang' or 'EMP-4507')",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n jira_data = None\n checked_paths = []\n\n # Try external_services_path first\n if external_services_path:\n for candidate in [\n Path(external_services_path) / \"jira_state.json\",\n Path(external_services_path) / \"final\" / \"jira_state.json\",\n ]:\n checked_paths.append(str(candidate))\n if candidate.exists():\n try:\n with open(candidate, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n break\n except Exception:\n pass\n\n # Also try workspace_path\n if jira_data is None:\n for candidate in [\n Path(workspace_path) / \"jira_state.json\",\n Path(workspace_path) / \"external_data\" / \"final\" / \"jira_state.json\",\n ]:\n checked_paths.append(str(candidate))\n if candidate.exists():\n try:\n with open(candidate, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n break\n except Exception:\n pass\n\n # Try glob as last resort\n if jira_data is None:\n for p in Path(workspace_path).rglob(\"jira_state.json\"):\n checked_paths.append(str(p))\n try:\n with open(p, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n break\n except Exception:\n pass\n\n if jira_data is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"jira_state.json not found. Checked: {checked_paths}\"}\n\n # Extract issues from the top-level 'issues' object\n issues = []\n if isinstance(jira_data, dict):\n if \"issues\" in jira_data:\n raw_issues = jira_data[\"issues\"]\n if isinstance(raw_issues, list):\n issues = raw_issues\n elif isinstance(raw_issues, dict):\n issues = list(raw_issues.values())\n else:\n for k, v in jira_data.items():\n if isinstance(v, dict):\n issues.append(v)\n elif isinstance(jira_data, list):\n issues = jira_data\n\n if not issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No issues found in jira_state.json. Top-level keys: {list(jira_data.keys()) if isinstance(jira_data, dict) else 'N/A (list)'}\"}\n\n def extract_text_from_description(desc):\n \"\"\"Extract all text from a description field, handling both string and ADF document objects.\"\"\"\n if isinstance(desc, str):\n return desc\n if isinstance(desc, dict):\n # Atlassian Document Format (ADF) - nested content[].content[].text\n texts = []\n _extract_text_recursive(desc, texts)\n return \" \".join(texts)\n return \"\"\n\n def _extract_text_recursive(node, texts):\n \"\"\"Recursively extract text from an ADF node.\"\"\"\n if isinstance(node, dict):\n if \"text\" in node and isinstance(node[\"text\"], str):\n texts.append(node[\"text\"])\n if \"content\" in node and isinstance(node[\"content\"], list):\n for child in node[\"content\"]:\n _extract_text_recursive(child, texts)\n # Also check other possible nested structures\n for key in [\"value\", \"body\"]:\n if key in node:\n _extract_text_recursive(node[key], texts)\n elif isinstance(node, list):\n for item in node:\n _extract_text_recursive(item, texts)\n\n def get_summary(issue):\n \"\"\"Get summary from an issue, checking nested 'fields' dict and flat keys.\"\"\"\n val = \"\"\n if \"fields\" in issue and isinstance(issue[\"fields\"], dict
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775880364263",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "In external_data/final/jira_state.json, the issue keyed 'OPS-5' under the top-level 'issues' object must NOT contain any comment where the text contains ('Foundry Bistro' or 'Capital Grille' or 'Harvest Kitchen') AND contains 'Missing business justification'",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Rubric: In external_data/final/jira_state.json, the issue keyed 'OPS-5'\n # under the top-level 'issues' object must NOT contain any comment where the text\n # contains ('Foundry Bistro' or 'Capital Grille' or 'Harvest Kitchen') AND contains\n # 'Missing business justification'\n\n jira_data = None\n checked_paths = []\n\n # Build candidate paths in order of priority\n candidates = []\n\n # The rubric says \"external_data/final/jira_state.json\" which is relative to workspace\n candidates.append(Path(workspace_path) / \"external_data\" / \"final\" / \"jira_state.json\")\n candidates.append(Path(workspace_path) / \"jira_state.json\")\n\n # Also try external_services_path (jira is an external service)\n if external_services_path is not None:\n candidates.insert(0, Path(external_services_path) / \"jira_state.json\")\n\n for p in candidates:\n checked_paths.append(str(p))\n if p.exists():\n try:\n with open(p, \"r\", encoding=\"utf-8\") as f:\n jira_data = json.load(f)\n break\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse {p}: {e}\"}\n\n if jira_data is None:\n # All three runs failed, so we should not pass by default.\n # The file not being found likely means the agent didn't produce expected output.\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"jira_state.json not found at any of: {checked_paths}. Cannot verify the rubric without this file.\"}\n\n target_key = \"OPS-5\"\n issue = None\n\n # The rubric says: \"the issue keyed 'OPS-5' under the top-level 'issues' object\"\n if \"issues\" in jira_data:\n issues_obj = jira_data[\"issues\"]\n if isinstance(issues_obj, dict):\n if target_key in issues_obj:\n issue = issues_obj[target_key]\n elif isinstance(issues_obj, list):\n for iss in issues_obj:\n if isinstance(iss, dict):\n key = iss.get(\"key\") or iss.get(\"issue_key\") or iss.get(\"id\") or \"\"\n if str(key) == target_key:\n issue = iss\n break\n\n # Fallback: top-level key\n if issue is None and target_key in jira_data:\n issue = jira_data[target_key]\n\n # Fallback: recursive search\n if issue is None:\n def find_issue(obj, target):\n if isinstance(obj, dict):\n if str(obj.get(\"key\", \"\")) == target or str(obj.get(\"issue_key\", \"\")) == target or str(obj.get(\"id\", \"\")) == target:\n return obj\n for v in obj.values():\n result = find_issue(v, target)\n if result is not None:\n return result\n elif isinstance(obj, list):\n for item in obj:\n result = find_issue(item, target)\n if result is not None:\n return result\n return None\n issue = find_issue(jira_data, target_key)\n\n if issue is None:\n # Issue not found - since this is checking a negative (must NOT contain),\n # if the issue doesn't exist, the forbidden comment can't exist.\n # But since all 3 runs failed, this likely means the agent didn't process correctly.\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue {target_key} not found in jira_state.json. The agent may not have processed the T&E reports correctly.\"}\n\n # Now look for comments on this issue\n comments = []\n if isinstance(issue, dict):\n # Try multiple possible comment field locations\n for ckey in [\"comments\", \"comment\", \"fields\"]:\n if ckey in issue:\n val = issue[ckey]\n if ckey == \"fields\" and i
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775880383123",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "In `slack_data.json`, there must be at least one message in a channel where `is_im` is true and the channel's `user` field equals `U002`, where that message contains both `[T&E Review Flag]` and `EMP-3312`.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n candidates = []\n \n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for candidate in candidates:\n if candidate.exists():\n slack_path = candidate\n break\n \n if slack_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found in workspace or external services path.\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n channels_obj = slack_data.get('channels', {})\n messages_obj = slack_data.get('messages', {})\n \n matching_messages = []\n \n for channel_id, channel in channels_obj.items():\n if not isinstance(channel, dict):\n continue\n \n is_im = channel.get('is_im', False)\n if is_im is not True:\n continue\n \n user_field = str(channel.get('user', '')).strip()\n if user_field != 'U002':\n continue\n \n channel_messages = messages_obj.get(channel_id, [])\n if not isinstance(channel_messages, list):\n continue\n \n for msg in channel_messages:\n if isinstance(msg, dict):\n text = ''\n for field in ['text', 'message', 'content', 'body']:\n if field in msg:\n text += ' ' + str(msg[field])\n msg_str = json.dumps(msg)\n elif isinstance(msg, str):\n text = msg\n msg_str = msg\n else:\n continue\n \n has_flag = '[T&E Review Flag]' in text or '[T&E Review Flag]' in msg_str\n has_emp = 'EMP-3312' in text or 'EMP-3312' in msg_str\n \n if has_flag and has_emp:\n matching_messages.append(msg_str[:200])\n \n if matching_messages:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found {len(matching_messages)} matching message(s) in a DM channel with user U002 containing both '[T&E Review Flag]' and 'EMP-3312'. First match: {matching_messages[0]}\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"No message found in a DM channel (is_im=true, user=U002) containing both '[T&E Review Flag]' and 'EMP-3312'. Checked all channels in slack_data.json.\"\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1775880399839",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "In `slack_data.json`, there must NOT be any message in any channel where `is_im` is true whose message text contains '[T&E Review Flag]' AND contains at least one of 'OPS-4', 'EMP-4507', or 'Derek Huang'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_path = None\n \n candidate = Path(workspace_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None and external_services_path is not None:\n candidate = Path(external_services_path) / \"slack_data.json\"\n if candidate.exists():\n slack_path = candidate\n \n if slack_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No slack_data.json found; condition is trivially satisfied (no violating messages possible).\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n violating_messages = []\n keywords = ['OPS-4', 'EMP-4507', 'Derek Huang']\n flag_text = '[T&E Review Flag]'\n \n channels_obj = slack_data.get('channels', {})\n messages_obj = slack_data.get('messages', {})\n \n for channel_id, channel in channels_obj.items():\n if not isinstance(channel, dict):\n continue\n \n is_im = channel.get(\"is_im\", False)\n if is_im is not True:\n continue\n \n channel_messages = messages_obj.get(channel_id, [])\n if not isinstance(channel_messages, list):\n continue\n \n for msg in channel_messages:\n if not isinstance(msg, dict):\n continue\n text = msg.get(\"text\", \"\") or \"\"\n if not isinstance(text, str):\n text = str(text)\n \n if flag_text in text:\n for kw in keywords:\n if kw.lower() in text.lower():\n violating_messages.append({\n \"channel\": channel.get(\"name\", channel_id),\n \"text_snippet\": text[:200],\n \"matched_keyword\": kw\n })\n break\n \n if violating_messages:\n details = \"; \".join(\n f\"Channel '{v['channel']}': matched '{v['matched_keyword']}' in message: '{v['text_snippet']}'\" \n for v in violating_messages\n )\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(violating_messages)} violating message(s) in IM channels containing '[T&E Review Flag]' and a prohibited keyword. Details: {details}\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No messages found in IM channels that contain both '[T&E Review Flag]' and any of 'OPS-4', 'EMP-4507', or 'Derek Huang'.\"\n }",
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776019924720",
|
||
|
|
"sort_order": 12,
|
||
|
|
"rubric_text": "In external_data/final/jira_state.json, issue OPS-5 must have at least one comment (stored either in the issue's own comment field or in the top-level comments collection) where the text contains T&E review complete AND all lines supported AND OPS-5 (case-insensitive; em-dash or hyphen accepted as separator between phrases).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n candidates = [\n Path(workspace_path) / \"external_data\" / \"final\" / \"jira_state.json\",\n ]\n if external_services_path:\n candidates.append(Path(external_services_path) / \"final\" / \"jira_state.json\")\n candidates.append(Path(external_services_path) / \"jira_state.json\")\n\n jira_path = None\n for c in candidates:\n if c.exists():\n jira_path = c\n break\n\n if jira_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find jira_state.json. Checked: {[str(c) for c in candidates]}\"}\n\n try:\n with open(jira_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse jira_state.json: {e}\"}\n\n issue_key = \"OPS-5\"\n issues = data.get(\"issues\", {})\n issue = issues.get(issue_key)\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue '{issue_key}' not found in issues object.\"}\n\n # Collect comments from BOTH locations:\n # 1) inside the issue object (flat or nested ADF)\n # 2) the top-level comments dict keyed by issue ID\n comments = []\n\n if isinstance(issue, dict):\n if \"comments\" in issue and isinstance(issue[\"comments\"], list):\n comments.extend(issue[\"comments\"])\n elif \"fields\" in issue and isinstance(issue[\"fields\"], dict):\n comment_field = issue[\"fields\"].get(\"comment\", {})\n if isinstance(comment_field, dict) and \"comments\" in comment_field:\n comments.extend(comment_field[\"comments\"])\n elif isinstance(comment_field, list):\n comments.extend(comment_field)\n\n # Top-level comments collection\n top_level_comments = data.get(\"comments\", {})\n if isinstance(top_level_comments, dict):\n bucket = top_level_comments.get(issue_key)\n if isinstance(bucket, list):\n comments.extend(bucket)\n elif isinstance(bucket, dict):\n # In case it's wrapped, e.g. {\"comments\": [...]}\n inner = bucket.get(\"comments\")\n if isinstance(inner, list):\n comments.extend(inner)\n elif isinstance(top_level_comments, list):\n # Fallback: a flat list where each entry tags its issue\n for c in top_level_comments:\n if isinstance(c, dict) and c.get(\"issue\") == issue_key:\n comments.append(c)\n\n if not comments:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue '{issue_key}' has no comments in either the issue object or the top-level comments collection.\"}\n\n def extract_text(node):\n \"\"\"Recursively extract text from an Atlassian Document Format node or plain string.\"\"\"\n if isinstance(node, str):\n return node\n if isinstance(node, dict):\n texts = []\n if \"text\" in node and isinstance(node[\"text\"], str):\n texts.append(node[\"text\"])\n for key in (\"content\", \"body\", \"value\"):\n if key in node:\n texts.append(extract_text(node[key]))\n return \" \".join(texts)\n if isinstance(node, list):\n return \" \".join(extract_text(item) for item in node)\n return \"\"\n\n def get_comment_text(comment):\n \"\"\"Get text from a comment object, handling both plain strings and ADF bodies.\"\"\"\n if isinstance(comment, str):\n return comment\n if isinstance(comment, dict):\n body = comment.get(\"body\")\n if body is not None:\n return extract_text(body)\n for field in (\"text\", \"content\", \"message\"):\n val = comment.get(field)\n if isinstance(val, str):\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776019964232",
|
||
|
|
"sort_order": 13,
|
||
|
|
"rubric_text": "In `slack_data.json`, there must NOT be any message in any channel where `is_im` is true whose message text contains '[T&E Review Flag]' AND contains at least one of 'OPS-5', 'EMP-5021', or 'Vanessa Frost'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # We need to check external services for slack_data.json\n # It could be in workspace or external services path\n slack_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"slack_data.json\")\n candidates.append(Path(workspace_path) / \"slack_data.json\")\n \n for c in candidates:\n if c.exists():\n slack_path = c\n break\n \n if slack_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"slack_data.json not found in any expected location. Since no Slack data exists, the negative condition is trivially satisfied (no violating messages exist).\"}\n \n try:\n with open(slack_path, 'r', encoding='utf-8') as f:\n slack_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {e}\"}\n \n # Navigate the slack data structure\n # Typical structures: {\"channels\": [...]} or {\"ims\": [...]} or list of channels\n channels = []\n if isinstance(slack_data, dict):\n # Could have 'channels', 'ims', or be keyed by channel id\n if 'channels' in slack_data:\n channels = slack_data['channels'] if isinstance(slack_data['channels'], list) else list(slack_data['channels'].values())\n if 'ims' in slack_data:\n ims = slack_data['ims'] if isinstance(slack_data['ims'], list) else list(slack_data['ims'].values())\n channels.extend(ims)\n # If neither, try treating all values as channels\n if not channels:\n for key, val in slack_data.items():\n if isinstance(val, (dict, list)):\n if isinstance(val, dict):\n channels.append(val)\n elif isinstance(val, list):\n channels.extend(val)\n elif isinstance(slack_data, list):\n channels = slack_data\n \n violating_messages = []\n keywords = ['OPS-5', 'EMP-5021', 'Vanessa Frost']\n keywords_lower = [k.lower() for k in keywords]\n flag_text = '[T&E Review Flag]'.lower()\n \n def check_messages(messages, channel_info=\"\"):\n if not isinstance(messages, list):\n return\n for msg in messages:\n if not isinstance(msg, dict):\n continue\n text = msg.get('text', '') or msg.get('message', '') or msg.get('content', '') or ''\n text_lower = text.lower()\n if flag_text in text_lower:\n for i, kw_lower in enumerate(keywords_lower):\n if kw_lower in text_lower:\n violating_messages.append({\n 'channel_info': channel_info,\n 'keyword': keywords[i],\n 'text_snippet': text[:200]\n })\n break\n \n for ch in channels:\n if not isinstance(ch, dict):\n continue\n is_im = ch.get('is_im', False)\n if is_im is True or is_im == 'true' or str(is_im).lower() == 'true':\n # This is an IM channel - check its messages\n msgs = ch.get('messages', []) or ch.get('history', []) or []\n ch_name = ch.get('name', ch.get('id', 'unknown'))\n check_messages(msgs, channel_info=f\"IM channel: {ch_name}\")\n \n if violating_messages:\n details = \"; \".join([f\"Found '{v['keyword']}' with '[T&E Review Flag]' in {v['channel_info']}: \\\"{v['text_snippet']}...\\\"\" for v in violating_messages])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Found {len(violating_messages)} violating message(s) in IM channels. {details}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"PASS: No messages found in IM channels containing '[T&E Review Flag]' along with
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
}
|
||
|
|
]
|