DerekSurge
115 lines
52 KiB
JSON
115 lines
52 KiB
JSON
[
|
|
{
|
|
"id": "b46d7e62-0c85-4abc-954e-fac4dcdd21f7",
|
|
"sort_order": 0,
|
|
"rubric_text": "In `jira_state.json`, issue OPS_HR_TE-2 must have a comment containing 'T&E review complete—all lines supported' and 'OPS_HR_TE-2'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"jira_state.json not found at {jira_path}\"}\n\n with open(jira_path, \"r\") as f:\n data = json.load(f)\n\n # Navigate to the issues dict (may be under a top-level \"data\" wrapper)\n if \"data\" in data and isinstance(data[\"data\"], dict):\n data = data[\"data\"]\n\n issues = data.get(\"issues\", {})\n issue = issues.get(\"OPS_HR_TE-2\")\n if issue is None:\n for v in issues.values():\n if isinstance(v, dict) and (v.get(\"key\") == \"OPS_HR_TE-2\" or v.get(\"id\") == \"OPS_HR_TE-2\"):\n issue = v\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Issue OPS_HR_TE-2 not found in jira_state.json\"}\n\n # Get comments — stored at top-level data[\"comments\"][\"OPS_HR_TE-2\"], not inside the issue\n raw_comments = data.get(\"comments\", {}).get(\"OPS_HR_TE-2\", [])\n\n def extract_text(body):\n \"\"\"Extract plain text from an Atlassian document body (dict) or plain string.\"\"\"\n if isinstance(body, str):\n return body\n if isinstance(body, dict):\n parts = []\n for block in body.get(\"content\", []):\n for inline in block.get(\"content\", []):\n t = inline.get(\"text\", \"\")\n if t:\n parts.append(t)\n return \" \".join(parts)\n return \"\"\n\n all_comment_texts = [extract_text(c.get(\"body\", \"\")) for c in raw_comments if isinstance(c, dict)]\n all_comment_texts += [c for c in raw_comments if isinstance(c, str)]\n\n all_comment_text = \" \".join(all_comment_texts)\n all_comment_lower = all_comment_text.lower()\n normalized = all_comment_lower.replace('\\u2014', '-').replace('\\u2013', '-').replace('--', '-')\n\n has_te_review = 't&e review complete' in all_comment_lower or 't&e review complete' in normalized\n has_all_lines = 'all lines supported' in all_comment_lower or 'all lines supported' in normalized\n\n if not has_te_review or not has_all_lines:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"OPS_HR_TE-2 missing required comment containing 'T&E review complete—all lines supported'. Found comments: {all_comment_texts}\"}\n\n if \"ops-hr-te-4501\" not in all_comment_lower:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"OPS_HR_TE-2 comment missing reference to 'OPS_HR_TE-2'. Found comments: {all_comment_texts}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"OPS_HR_TE-2 contains a comment with 'T&E review complete—all lines supported' and 'OPS_HR_TE-2'. Comments found: {all_comment_texts}\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "0d769e20-3f9e-4232-ac90-a94d44757d50",
|
|
"sort_order": 1,
|
|
"rubric_text": "In `jira_state.json`, issue OPS_HR_TE-3 must have a comment containing 'Missing business justification. Please submit personal reimbursement for line item: 1.'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"jira_state.json not found at {jira_path}\"}\n\n with open(jira_path, \"r\") as f:\n data = json.load(f)\n\n # Navigate to the issues dict (may be under a top-level \"data\" wrapper)\n if \"data\" in data and isinstance(data[\"data\"], dict):\n data = data[\"data\"]\n\n issues = data.get(\"issues\", {})\n issue = issues.get(\"OPS_HR_TE-3\")\n if issue is None:\n for v in issues.values():\n if isinstance(v, dict) and (v.get(\"key\") == \"OPS_HR_TE-3\" or v.get(\"id\") == \"OPS_HR_TE-3\"):\n issue = v\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Issue OPS_HR_TE-3 not found in jira_state.json\"}\n\n # Get comments — stored at top-level data[\"comments\"][\"OPS_HR_TE-3\"], not inside the issue\n raw_comments = data.get(\"comments\", {}).get(\"OPS_HR_TE-3\", [])\n\n def extract_text(body):\n \"\"\"Extract plain text from an Atlassian document body (dict) or plain string.\"\"\"\n if isinstance(body, str):\n return body\n if isinstance(body, dict):\n parts = []\n for block in body.get(\"content\", []):\n for inline in block.get(\"content\", []):\n t = inline.get(\"text\", \"\")\n if t:\n parts.append(t)\n return \" \".join(parts)\n return \"\"\n\n all_comment_texts = [extract_text(c.get(\"body\", \"\")) for c in raw_comments if isinstance(c, dict)]\n all_comment_texts += [c for c in raw_comments if isinstance(c, str)]\n\n all_comment_text_lower = \" \".join(all_comment_texts).lower()\n\n has_missing_biz_just = \"missing business justification\" in all_comment_text_lower\n has_personal_reimb_line1 = (\n \"please submit personal reimbursement\" in all_comment_text_lower\n and \"line item\" in all_comment_text_lower\n and \"1\" in all_comment_text_lower\n )\n\n if not has_missing_biz_just:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"OPS_HR_TE-3 missing required comment phrase 'Missing business justification'. Found comments: {all_comment_texts}\"}\n\n if not has_personal_reimb_line1:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"OPS_HR_TE-3 missing required comment phrase about personal reimbursement for line item 1. Found comments: {all_comment_texts}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"OPS_HR_TE-3 contains the required comment about missing business justification and personal reimbursement for line item 1. Comments found: {all_comment_texts}\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "4bf7d578-d098-440d-bb0c-ed41ed11cd2c",
|
|
"sort_order": 2,
|
|
"rubric_text": "In `jira_state.json`, issue OPS_HR_TE-4 must have a comment containing 'Unauthorized vendor. Please submit personal reimbursement for line item: 1.'",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"jira_state.json not found at {jira_path}\"}\n\n with open(jira_path, \"r\") as f:\n data = json.load(f)\n\n # Navigate to the issues dict (may be under a top-level \"data\" wrapper)\n if \"data\" in data and isinstance(data[\"data\"], dict):\n data = data[\"data\"]\n\n issues = data.get(\"issues\", {})\n issue = issues.get(\"OPS_HR_TE-4\")\n if issue is None:\n for v in issues.values():\n if isinstance(v, dict) and (v.get(\"key\") == \"OPS_HR_TE-4\" or v.get(\"id\") == \"OPS_HR_TE-4\"):\n issue = v\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Issue OPS_HR_TE-4 not found in jira_state.json\"}\n\n # Get comments — stored at top-level data[\"comments\"][\"OPS_HR_TE-4\"], not inside the issue\n raw_comments = data.get(\"comments\", {}).get(\"OPS_HR_TE-4\", [])\n\n def extract_text(body):\n \"\"\"Extract plain text from an Atlassian document body (dict) or plain string.\"\"\"\n if isinstance(body, str):\n return body\n if isinstance(body, dict):\n parts = []\n for block in body.get(\"content\", []):\n for inline in block.get(\"content\", []):\n t = inline.get(\"text\", \"\")\n if t:\n parts.append(t)\n return \" \".join(parts)\n return \"\"\n\n all_comment_texts = [extract_text(c.get(\"body\", \"\")) for c in raw_comments if isinstance(c, dict)]\n all_comment_texts += [c for c in raw_comments if isinstance(c, str)]\n\n all_comment_text = \" \".join(all_comment_texts).lower()\n\n required_phrase_1 = \"unauthorized vendor\"\n required_phrase_2 = \"please submit personal reimbursement for line item\"\n\n has_phrase_1 = required_phrase_1 in all_comment_text\n has_phrase_2 = required_phrase_2 in all_comment_text and \"1\" in all_comment_text\n\n if not has_phrase_1 or not has_phrase_2:\n missing = []\n if not has_phrase_1:\n missing.append(f\"'{required_phrase_1}'\")\n if not has_phrase_2:\n missing.append(\"'please submit personal reimbursement for line item: 1'\")\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"OPS_HR_TE-4 missing required comment content: {', '.join(missing)}. Found comments: {all_comment_texts}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"OPS_HR_TE-4 contains required comment with 'Unauthorized vendor. Please submit personal reimbursement for line item: 1.'\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "e55c834e-aa4a-48f9-acb2-9005dfc9d1ed",
|
|
"sort_order": 3,
|
|
"rubric_text": "In `jira_state.json`, issue OPS_HR_TE-5 must have a comment containing 'Missing business justification. Please submit personal reimbursement for line items: 2, 3.'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate jira_state.json\"}\n\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"jira_state.json not found at {jira_path}\"}\n\n with open(jira_path, \"r\") as f:\n data = json.load(f)\n\n # Navigate to the issues dict (may be under a top-level \"data\" wrapper)\n if \"data\" in data and isinstance(data[\"data\"], dict):\n data = data[\"data\"]\n\n issues = data.get(\"issues\", {})\n issue = issues.get(\"OPS_HR_TE-5\")\n if issue is None:\n for k, v in issues.items():\n if isinstance(v, dict) and (v.get(\"key\") == \"OPS_HR_TE-5\" or v.get(\"id\") == \"OPS_HR_TE-5\"):\n issue = v\n break\n\n if issue is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Issue OPS_HR_TE-5 not found in jira_state.json\"}\n\n # Get comments — stored at top-level data[\"comments\"][\"OPS_HR_TE-5\"], not inside the issue\n raw_comments = data.get(\"comments\", {}).get(\"OPS_HR_TE-5\", [])\n\n def extract_text(body):\n \"\"\"Extract plain text from an Atlassian document body (dict) or plain string.\"\"\"\n if isinstance(body, str):\n return body\n if isinstance(body, dict):\n parts = []\n for block in body.get(\"content\", []):\n for inline in block.get(\"content\", []):\n t = inline.get(\"text\", \"\")\n if t:\n parts.append(t)\n return \" \".join(parts)\n return \"\"\n\n all_comment_texts = [extract_text(c.get(\"body\", \"\")) for c in raw_comments if isinstance(c, dict)]\n all_comment_texts += [c for c in raw_comments if isinstance(c, str)]\n\n all_comment_text = \" \".join(all_comment_texts).lower()\n\n required_part1 = \"missing business justification\"\n required_part2_variants = [\n \"please submit personal reimbursement for line items: 2, 3\",\n \"please submit personal reimbursement for line items: 2,3\",\n \"please submit personal reimbursement for line items 2, 3\",\n \"please submit personal reimbursement for line items 2,3\",\n ]\n\n if required_part1 not in all_comment_text:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"OPS_HR_TE-5 missing required comment containing 'Missing business justification'. Found comments: {all_comment_texts}\"}\n\n found_part2 = any(variant in all_comment_text for variant in required_part2_variants)\n if not found_part2:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"OPS_HR_TE-5 missing required comment containing 'Please submit personal reimbursement for line items: 2, 3'. Found comments: {all_comment_texts}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"OPS_HR_TE-5 has the correct comment containing 'Missing business justification. Please submit personal reimbursement for line items: 2, 3.'\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "64bdfe1e-7e10-4c7d-a6f1-20f4b80b1dd5",
|
|
"sort_order": 4,
|
|
"rubric_text": "In `slack_data.json`, channel D002 must contain a message with '[T&E Review Flag]' for Employee ID EMP-2103, referencing expense date around March 28 2026, amount $215.00, merchant Capital Grille, noting missing calendar event, and stating action taken in OPS_HR_TE workflow.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found at {slack_path}\"}\n \n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n messages = data.get(\"messages\", {})\n d002_messages = messages.get(\"D002\", [])\n \n if not d002_messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in Slack channel D002\"}\n \n required_elements = [\n \"t&e review flag\",\n \"emp-2103\",\n \"215\",\n \"capital grille\",\n \"calendar\",\n \"ops-hr-te\"\n ]\n \n for msg in d002_messages:\n text = msg.get(\"text\", \"\").lower()\n # Check for march 28 date in various formats\n date_present = any(x in text for x in [\"march 28\", \"mar 28\", \"03/28\", \"3/28\", \"28 march\", \"march 28th\"])\n if all(elem in text for elem in required_elements) and date_present:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found required T&E Review Flag message in D002 for EMP-2103 with all required details\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No message in D002 found containing all required elements: [T&E Review Flag], EMP-2103, $215, Capital Grille, calendar event reference, OPS_HR_TE, and March 28 2026 date\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "c271f097-dc6c-48a0-966b-78e49e1da33f",
|
|
"sort_order": 5,
|
|
"rubric_text": "In `slack_data.json`, channel D001 must contain a message with '[T&E Review Flag]' for Employee ID EMP-1592, referencing expense dates around April 3-4 2026, amounts $387.00 and $245.00, merchants Delta Airlines and Hilton Garden Inn SFO, noting missing calendar event, and stating action taken in OPS_HR_TE workflow.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found at {slack_path}\"}\n \n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n messages = data.get(\"messages\", {})\n d001_messages = messages.get(\"D001\", [])\n \n if not d001_messages:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in Slack channel D001\"}\n \n required_elements = [\n \"t&e review flag\",\n \"emp-1592\",\n \"387\",\n \"245\",\n \"delta airlines\",\n \"hilton garden inn\",\n \"calendar\",\n \"ops-hr-te\"\n ]\n \n for msg in d001_messages:\n text = msg.get(\"text\", \"\").lower()\n date_present = any(x in text for x in [\"april 3\", \"apr 3\", \"04/03\", \"4/3\", \"april 3rd\"])\n if all(elem in text for elem in required_elements) and date_present:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found required T&E Review Flag message in D001 for EMP-1592 with all required details\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No message in D001 found containing all required elements: [T&E Review Flag], EMP-1592, $387, $245, Delta Airlines, Hilton Garden Inn SFO, calendar event reference, OPS_HR_TE, April 3-4 2026 date\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "751f72cb-45cc-4cac-a307-5cfc3c4f2b56",
|
|
"sort_order": 6,
|
|
"rubric_text": "In `slack_data.json`, no message in channel D001 or D002 must contain 'EMP-1847' or 'EMP-2287'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found at {slack_path}\"}\n \n with open(slack_path, \"r\") as f:\n data = json.load(f)\n \n messages = data.get(\"messages\", {})\n forbidden_ids = [\"EMP-1847\", \"EMP-2287\"]\n \n for channel_id in [\"D001\", \"D002\"]:\n channel_messages = messages.get(channel_id, [])\n for msg in channel_messages:\n text = msg.get(\"text\", \"\")\n for forbidden_id in forbidden_ids:\n if forbidden_id.lower() in text.lower():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found forbidden employee ID '{forbidden_id}' in message in channel {channel_id}: {text[:200]}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No messages in D001 or D002 contain forbidden employee IDs EMP-1847 or EMP-2287\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "19603a8e-b441-4e74-aba4-49125278877b",
|
|
"sort_order": 7,
|
|
"rubric_text": "In `mailbox.json`, a sent email must exist addressed to people@vanguardshield.com containing '[Reporting Manager Lookup]', employee 'Derek Washington', ID 'EMP-2287', and issue 'OPS_HR_TE-4', requesting reporting manager name and Jira username.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n required_elements = [\n \"reporting manager lookup\",\n \"derek washington\",\n \"emp-2287\",\n \"ops-hr-te-4503\",\n \"reporting manager\"\n ]\n \n for email in emails:\n to_addr = (email.get(\"to_addr\", \"\") or \"\").lower()\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n combined = body + \" \" + subject\n folder = (email.get(\"folder\", \"\") or \"\").lower()\n \n if \"people@vanguardshield.com\" not in to_addr:\n continue\n \n if all(elem in combined for elem in required_elements):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found required email to people@vanguardshield.com with [Reporting Manager Lookup] for Derek Washington / EMP-2287 / OPS_HR_TE-4\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email found to people@vanguardshield.com containing all required elements: [Reporting Manager Lookup], Derek Washington, EMP-2287, OPS_HR_TE-4\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "79b187e8-1b17-451a-88a9-85537873094f",
|
|
"sort_order": 8,
|
|
"rubric_text": "In `mailbox.json`, no email must contain 'OPS_HR_TE-2', 'OPS_HR_TE-3', or 'OPS_HR_TE-5' in the subject or body.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot locate mailbox.json\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n forbidden_issues = [\"OPS_HR_TE-2\", \"OPS_HR_TE-3\", \"OPS_HR_TE-5\"]\n \n for email in emails:\n body = (email.get(\"body_text\", \"\") or \"\").lower()\n subject = (email.get(\"subject\", \"\") or \"\").lower()\n combined = body + \" \" + subject\n \n for forbidden in forbidden_issues:\n if forbidden.lower() in combined:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found forbidden issue reference '{forbidden}' in email (subject: {email.get('subject', '')})\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No emails contain forbidden issue references OPS_HR_TE-2, OPS_HR_TE-3, or OPS_HR_TE-5\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776104758013",
|
|
"sort_order": 9,
|
|
"rubric_text": "In `jira_state.json`, issue OPS_HR_TE-2 must NOT have comments containing 'Missing business justification' or 'unauthorized vendor'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check both workspace and external_services for jira_state.json\n jira_path = None\n candidates = []\n \n if workspace_path:\n candidates.append(Path(workspace_path) / \"jira_state.json\")\n if external_services_path:\n candidates.append(Path(external_services_path) / \"jira_state.json\")\n \n for c in candidates:\n if c.exists():\n jira_path = c\n break\n \n if jira_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"jira_state.json not found in workspace or external services. Since we cannot confirm the comments exist, passing by default (prefer false positives).\"}\n \n try:\n with open(jira_path, 'r') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not parse jira_state.json: {e}. Passing by default.\"}\n \n # Find issue OPS_HR_TE-2\n issue = None\n \n # Handle different possible structures\n if isinstance(jira_data, dict):\n # Could be keyed by issue id directly\n if \"OPS_HR_TE-2\" in jira_data:\n issue = jira_data[\"OPS_HR_TE-2\"]\n # Could be in an \"issues\" key\n elif \"issues\" in jira_data:\n issues = jira_data[\"issues\"]\n if isinstance(issues, dict):\n if \"OPS_HR_TE-2\" in issues:\n issue = issues[\"OPS_HR_TE-2\"]\n else:\n for k, v in issues.items():\n if isinstance(v, dict) and v.get(\"key\") == \"OPS_HR_TE-2\":\n issue = v\n break\n if isinstance(v, dict) and v.get(\"id\") == \"OPS_HR_TE-2\":\n issue = v\n break\n elif isinstance(issues, list):\n for item in issues:\n if isinstance(item, dict):\n if item.get(\"key\") == \"OPS_HR_TE-2\" or item.get(\"id\") == \"OPS_HR_TE-2\" or item.get(\"issue_id\") == \"OPS_HR_TE-2\":\n issue = item\n break\n # Try searching all top-level keys\n if issue is None:\n for k, v in jira_data.items():\n if isinstance(v, dict) and (v.get(\"key\") == \"OPS_HR_TE-2\" or v.get(\"id\") == \"OPS_HR_TE-2\"):\n issue = v\n break\n \n if issue is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Issue OPS_HR_TE-2 not found in jira_state.json. No offending comments can exist. Passing.\"}\n \n # Extract comments\n comments = []\n if isinstance(issue, dict):\n if \"comments\" in issue:\n raw_comments = issue[\"comments\"]\n if isinstance(raw_comments, list):\n comments = raw_comments\n elif isinstance(raw_comments, str):\n comments = [raw_comments]\n # Also check nested fields\n if \"fields\" in issue and isinstance(issue[\"fields\"], dict):\n fields = issue[\"fields\"]\n if \"comment\" in fields:\n comment_field = fields[\"comment\"]\n if isinstance(comment_field, dict) and \"comments\" in comment_field:\n comments.extend(comment_field[\"comments\"])\n elif isinstance(comment_field, list):\n comments.extend(comment_field)\n \n forbidden_phrases = ['missing business justification', 'unauthorized vendor']\n found_forbidden = []\n \n for comment in comments:\n comment_text = \"\"\n if isinstance(comment, str):\n comment_text = comment\n elif isinstance(comment, dict):\n # Try common keys for comment body\n for key in ['body', 'text', 'content', 'message', 'comment']:\n if key in comment and isinstance(comment[key], str):\n comment_text += \" \" + comment[key]\n \n comment_lower = comment_text.lower()\n for phrase in forbidden_phrases:\n if phrase in comment_lower:\n found_forbidden.append((phrase, comment_text[:100]))\n \n if found_forbidden:\n details = \"; \".join([f\"Found '{p}' in comment: '{t}...'\" for p, t in found_forbidden])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue OPS_HR_TE-2 has comments containing forbidden phrases. {details}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Issue OPS_HR_TE-2 does NOT contain comments with 'Missing business justification' or 'unauthorized vendor'. {len(comments)} comment(s) checked, all clean.\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776104819553",
|
|
"sort_order": 10,
|
|
"rubric_text": "In `jira_state.json`, issue OPS_HR_TE-3 must NOT have comments containing 'T&E review complete—all lines supported' or 'Unauthorized vendor'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check jira_state.json in external_services_path first, then workspace_path\n jira_file = None\n for base in [external_services_path, workspace_path]:\n if base is not None:\n candidate = Path(base) / \"jira_state.json\"\n if candidate.exists():\n jira_file = candidate\n break\n \n if jira_file is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"jira_state.json not found in any expected location. Since we cannot confirm the forbidden comments exist, passing by default (prefer false positives).\"}\n \n try:\n with open(jira_file, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not parse jira_state.json: {e}. Passing by default.\"}\n \n # Find issue OPS_HR_TE-3\n issue = None\n # Handle various possible structures\n if isinstance(data, dict):\n # Could be keyed by issue id directly\n if \"OPS_HR_TE-3\" in data:\n issue = data[\"OPS_HR_TE-3\"]\n # Could be in an 'issues' list or dict\n elif \"issues\" in data:\n issues = data[\"issues\"]\n if isinstance(issues, list):\n for iss in issues:\n if isinstance(iss, dict):\n key = iss.get(\"key\") or iss.get(\"id\") or iss.get(\"issue_key\") or iss.get(\"issue_id\") or \"\"\n if str(key) == \"OPS_HR_TE-3\":\n issue = iss\n break\n elif isinstance(issues, dict):\n if \"OPS_HR_TE-3\" in issues:\n issue = issues[\"OPS_HR_TE-3\"]\n # Could be in a 'tickets' list or dict\n elif \"tickets\" in data:\n tickets = data[\"tickets\"]\n if isinstance(tickets, list):\n for t in tickets:\n if isinstance(t, dict):\n key = t.get(\"key\") or t.get(\"id\") or t.get(\"issue_key\") or t.get(\"ticket_id\") or \"\"\n if str(key) == \"OPS_HR_TE-3\":\n issue = t\n break\n elif isinstance(tickets, dict):\n if \"OPS_HR_TE-3\" in tickets:\n issue = tickets[\"OPS_HR_TE-3\"]\n \n if issue is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Issue OPS_HR_TE-3 not found in jira_state.json. Since the forbidden comments cannot exist, passing.\"}\n \n # Extract comments\n comments = []\n if isinstance(issue, dict):\n raw_comments = issue.get(\"comments\") or issue.get(\"comment\") or []\n if isinstance(raw_comments, list):\n for c in raw_comments:\n if isinstance(c, str):\n comments.append(c)\n elif isinstance(c, dict):\n body = c.get(\"body\") or c.get(\"text\") or c.get(\"content\") or c.get(\"message\") or \"\"\n comments.append(str(body))\n elif isinstance(raw_comments, str):\n comments.append(raw_comments)\n \n forbidden_phrases = [\n \"t&e review complete\\u2014all lines supported\",\n \"t&e review complete—all lines supported\",\n \"t&e review complete-all lines supported\",\n \"unauthorized vendor\"\n ]\n \n found_forbidden = []\n for comment in comments:\n comment_lower = comment.lower().strip()\n for phrase in forbidden_phrases:\n if phrase.lower() in comment_lower:\n found_forbidden.append((comment, phrase))\n \n if found_forbidden:\n details = \"; \".join([f\"Comment '{c[:80]}...' matches forbidden phrase '{p}'\" for c, p in found_forbidden])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: Issue OPS_HR_TE-3 has comments containing forbidden text. {details}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PASS: Issue OPS_HR_TE-3 has {len(comments)} comment(s), none containing the forbidden phrases 'T&E review complete—all lines supported' or 'Unauthorized vendor'.\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776104871371",
|
|
"sort_order": 11,
|
|
"rubric_text": "In `jira_state.json`, issue OPS_HR_TE-4 must NOT have comments containing 'T&E review complete—all lines supported' or 'Missing business justification'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external services path for jira_state.json\n jira_path = None\n candidates = []\n \n # Check workspace\n if workspace_path:\n candidates.append(Path(workspace_path) / \"jira_state.json\")\n # Check external services\n if external_services_path:\n candidates.append(Path(external_services_path) / \"jira_state.json\")\n \n for c in candidates:\n if c.exists():\n jira_path = c\n break\n \n if jira_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"jira_state.json not found in workspace or external services. Since the file is absent, no offending comments exist. Passing by default (generous interpretation).\"}\n \n try:\n with open(jira_path, 'r', encoding='utf-8') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Could not parse jira_state.json: {e}. Passing by default (generous interpretation).\"}\n \n # Find issue OPS_HR_TE-4\n issue = None\n \n # Handle different possible structures\n if isinstance(jira_data, dict):\n # Could be keyed by issue id\n if \"OPS_HR_TE-4\" in jira_data:\n issue = jira_data[\"OPS_HR_TE-4\"]\n # Could have an 'issues' key\n elif \"issues\" in jira_data:\n issues = jira_data[\"issues\"]\n if isinstance(issues, dict) and \"OPS_HR_TE-4\" in issues:\n issue = issues[\"OPS_HR_TE-4\"]\n elif isinstance(issues, list):\n for iss in issues:\n if isinstance(iss, dict):\n key = iss.get(\"key\", iss.get(\"id\", iss.get(\"issue_key\", \"\")))\n if key == \"OPS_HR_TE-4\":\n issue = iss\n break\n # Could be a flat dict representing a single issue or have other keys\n else:\n # Try searching all values\n for k, v in jira_data.items():\n if isinstance(v, dict):\n key = v.get(\"key\", v.get(\"id\", v.get(\"issue_key\", \"\")))\n if key == \"OPS_HR_TE-4\":\n issue = v\n break\n elif isinstance(jira_data, list):\n for iss in jira_data:\n if isinstance(iss, dict):\n key = iss.get(\"key\", iss.get(\"id\", iss.get(\"issue_key\", \"\")))\n if key == \"OPS_HR_TE-4\":\n issue = iss\n break\n \n if issue is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Issue OPS_HR_TE-4 not found in jira_state.json. No offending comments can exist. Passing.\"}\n \n # Now check comments\n comments = []\n if isinstance(issue, dict):\n comments_raw = issue.get(\"comments\", issue.get(\"comment\", []))\n if isinstance(comments_raw, list):\n comments = comments_raw\n elif isinstance(comments_raw, dict):\n # Could be nested: {\"comments\": [...]}\n inner = comments_raw.get(\"comments\", [])\n if isinstance(inner, list):\n comments = inner\n else:\n comments = [comments_raw]\n \n forbidden_phrases = [\n \"t&e review complete—all lines supported\",\n \"t&e review complete\\u2014all lines supported\",\n \"t&e review complete-all lines supported\",\n \"missing business justification\"\n ]\n \n found_forbidden = []\n for comment in comments:\n comment_text = \"\"\n if isinstance(comment, str):\n comment_text = comment\n elif isinstance(comment, dict):\n comment_text = comment.get(\"body\", comment.get(\"text\", comment.get(\"content\", \"\")))\n if not isinstance(comment_text, str):\n comment_text = str(comment_text)\n else:\n comment_text = str(comment)\n \n lower_text = comment_text.lower().strip()\n for phrase in forbidden_phrases:\n if phrase in lower_text:\n found_forbidden.append(comment_text[:200])\n break\n \n if found_forbidden:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Issue OPS_HR_TE-4 has {len(found_forbidden)} forbidden comment(s). Found comments containing forbidden phrases: {found_forbidden}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"Issue OPS_HR_TE-4 does not contain comments with 'T&E review complete—all lines supported' or 'Missing business justification'. Rubric satisfied.\"\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776104913020",
|
|
"sort_order": 12,
|
|
"rubric_text": "In `jira_state.json`, no ticket must contain 'Manager Lookup'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # We need to find jira_state.json. It could be in workspace or external_services.\n jira_path = None\n \n # Check external_services_path first (Jira is typically an external service)\n if external_services_path:\n candidate = Path(external_services_path) / \"jira_state.json\"\n if candidate.exists():\n jira_path = candidate\n \n # Also check workspace_path\n if not jira_path:\n candidate = Path(workspace_path) / \"jira_state.json\"\n if candidate.exists():\n jira_path = candidate\n \n # Search recursively in both paths\n if not jira_path:\n for search_path in [external_services_path, workspace_path]:\n if search_path:\n found = list(Path(search_path).rglob(\"jira_state.json\"))\n if found:\n jira_path = found[0]\n break\n \n if not jira_path or not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find jira_state.json in workspace or external services path.\"}\n \n try:\n with open(jira_path, 'r', encoding='utf-8') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse jira_state.json: {e}\"}\n \n # The rubric says: no ticket must contain 'Manager Lookup' (anywhere in the ticket)\n search_term = 'Manager Lookup'\n violating_tickets = []\n \n # Handle various possible structures of jira_state.json\n tickets = []\n \n if isinstance(jira_data, list):\n tickets = jira_data\n elif isinstance(jira_data, dict):\n # Could be {\"tickets\": [...]}, {\"issues\": [...]}, or {ticket_id: {...}, ...}\n if \"tickets\" in jira_data:\n tickets = jira_data[\"tickets\"] if isinstance(jira_data[\"tickets\"], list) else list(jira_data[\"tickets\"].values())\n elif \"issues\" in jira_data:\n tickets = jira_data[\"issues\"] if isinstance(jira_data[\"issues\"], list) else list(jira_data[\"issues\"].values())\n else:\n # Might be a dict of ticket_id -> ticket_data\n for key, value in jira_data.items():\n if isinstance(value, dict):\n tickets.append(value)\n \n def check_value_for_term(val, term):\n \"\"\"Recursively check if a value (str, list, dict) contains the search term.\"\"\"\n if isinstance(val, str):\n return term in val\n elif isinstance(val, list):\n for item in val:\n if check_value_for_term(item, term):\n return True\n elif isinstance(val, dict):\n for k, v in val.items():\n if isinstance(k, str) and term in k:\n return True\n if check_value_for_term(v, term):\n return True\n return False\n \n for ticket in tickets:\n if not isinstance(ticket, dict):\n continue\n \n ticket_id = ticket.get('id', ticket.get('key', ticket.get('ticket_id', 'unknown')))\n \n # Check ALL fields in the ticket for 'Manager Lookup'\n for field_name, field_val in ticket.items():\n if check_value_for_term(field_val, search_term):\n # Summarize the value for feedback\n val_str = str(field_val)[:200]\n violating_tickets.append(f\"Ticket '{ticket_id}': field '{field_name}' contains '{search_term}' -> \\\"{val_str}\\\"\")\n break # One violation per ticket is enough\n \n if violating_tickets:\n details = \"\\n\".join(violating_tickets)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(violating_tickets)} ticket(s) containing 'Manager Lookup':\\n{details}\"\n }\n \n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Verified {len(tickets)} ticket(s) in jira_state.json. None contain 'Manager Lookup'.\"\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776104949399",
|
|
"sort_order": 13,
|
|
"rubric_text": "In `jira_state.json`, issue OPS_HR_TE-5 must NOT have comments containing 'T&E review complete—all lines supported' or 'Unauthorized vendor'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check jira_state.json in external services first, then workspace\n jira_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"jira_state.json\")\n candidates.append(Path(workspace_path) / \"jira_state.json\")\n # Also check subdirectories\n for p in Path(workspace_path).rglob(\"jira_state.json\"):\n candidates.append(p)\n if external_services_path:\n for p in Path(external_services_path).rglob(\"jira_state.json\"):\n candidates.append(p)\n\n jira_path = None\n for c in candidates:\n if c.exists():\n jira_path = c\n break\n\n if jira_path is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"jira_state.json not found anywhere. Since no comments could have been added, the condition is trivially satisfied (no forbidden comments exist).\"}\n\n try:\n with open(jira_path, 'r', encoding='utf-8') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse jira_state.json: {e}\"}\n\n # Find issue OPS_HR_TE-5\n issue = None\n # jira_state.json might be structured in different ways\n if isinstance(jira_data, dict):\n # Could be keyed by issue id\n if \"OPS_HR_TE-5\" in jira_data:\n issue = jira_data[\"OPS_HR_TE-5\"]\n # Could have an 'issues' key\n elif \"issues\" in jira_data:\n issues = jira_data[\"issues\"]\n if isinstance(issues, dict) and \"OPS_HR_TE-5\" in issues:\n issue = issues[\"OPS_HR_TE-5\"]\n elif isinstance(issues, list):\n for iss in issues:\n if isinstance(iss, dict):\n key = iss.get(\"key\") or iss.get(\"id\") or iss.get(\"issue_key\") or iss.get(\"issue_id\") or \"\"\n if key == \"OPS_HR_TE-5\":\n issue = iss\n break\n # Maybe flat dict with various issue keys\n else:\n for k, v in jira_data.items():\n if isinstance(v, dict):\n key = v.get(\"key\") or v.get(\"id\") or v.get(\"issue_key\") or \"\"\n if key == \"OPS_HR_TE-5\" or k == \"OPS_HR_TE-5\":\n issue = v\n break\n elif isinstance(jira_data, list):\n for iss in jira_data:\n if isinstance(iss, dict):\n key = iss.get(\"key\") or iss.get(\"id\") or iss.get(\"issue_key\") or iss.get(\"issue_id\") or \"\"\n if key == \"OPS_HR_TE-5\":\n issue = iss\n break\n\n if issue is None:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Issue OPS_HR_TE-5 not found in jira_state.json. No forbidden comments can exist.\"}\n\n # Gather all comments\n comments = []\n if isinstance(issue, dict):\n # Comments might be under various keys\n for ckey in [\"comments\", \"comment\", \"fields\"]:\n val = issue.get(ckey)\n if ckey == \"fields\" and isinstance(val, dict):\n val = val.get(\"comment\") or val.get(\"comments\")\n if val is not None:\n if isinstance(val, list):\n comments.extend(val)\n elif isinstance(val, dict) and \"comments\" in val:\n comments.extend(val[\"comments\"])\n elif isinstance(val, str):\n comments.append(val)\n\n # Extract comment text from each comment object\n forbidden_phrases = [\n \"t&e review complete\\u2014all lines supported\",\n \"t&e review complete—all lines supported\",\n \"t&e review complete-all lines supported\",\n \"unauthorized vendor\"\n ]\n\n found_forbidden = []\n for c in comments:\n text = \"\"\n if isinstance(c, str):\n text = c\n elif isinstance(c, dict):\n text = c.get(\"body\") or c.get(\"text\") or c.get(\"content\") or c.get(\"message\") or str(c)\n text_lower = text.lower()\n for phrase in forbidden_phrases:\n if phrase in text_lower:\n found_forbidden.append((phrase, text[:200]))\n\n if found_forbidden:\n details = \"; \".join([f\"Found '{p}' in comment: '{t}'\" for p, t in found_forbidden])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Issue OPS_HR_TE-5 has forbidden comments. {details}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Issue OPS_HR_TE-5 exists with {len(comments)} comment(s), none containing the forbidden phrases 'T&E review complete—all lines supported' or 'Unauthorized vendor'.\"}\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776105354420",
|
|
"sort_order": 14,
|
|
"rubric_text": "In `mailbox.json`, no email contains '[T&E Manager Lookup]' in its subject or body.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # This rubric checks mailbox.json for absence of '[T&E Manager Lookup]' in any email\n # mailbox.json could be in either workspace or external_services path\n \n mailbox_path = None\n \n # Check external_services_path first (most likely location)\n if external_services_path is not None:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find mailbox.json in workspace or external services path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not isinstance(emails, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json 'emails' field is not a list.\"}\n \n search_term = \"[T&E Manager Lookup]\"\n offending_emails = []\n \n for email in emails:\n subject = email.get(\"subject\", \"\") or \"\"\n body = email.get(\"body_text\", \"\") or \"\"\n email_id = email.get(\"email_id\", \"unknown\")\n \n if search_term in subject or search_term in body:\n offending_emails.append({\n \"email_id\": email_id,\n \"subject\": subject,\n \"found_in\": []\n })\n if search_term in subject:\n offending_emails[-1][\"found_in\"].append(\"subject\")\n if search_term in body:\n offending_emails[-1][\"found_in\"].append(\"body\")\n \n if len(offending_emails) == 0:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: Checked {len(emails)} emails in mailbox.json. None contain '[T&E Manager Lookup]'.\"\n }\n else:\n details = \"; \".join(\n f\"email_id={e['email_id']} (subject='{e['subject']}', found in: {', '.join(e['found_in'])})\"\n for e in offending_emails\n )\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found {len(offending_emails)} email(s) containing '[T&E Manager Lookup]': {details}\"\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776105407028",
|
|
"sort_order": 15,
|
|
"rubric_text": "In `mailbox.json`, there must be exactly 2 sent emails.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path first\n mailbox_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace_path as fallback\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in either external_services_path or workspace_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not isinstance(emails, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"'emails' field in mailbox.json is not a list.\"}\n \n sent_emails = [e for e in emails if isinstance(e, dict) and e.get(\"folder\", \"\").lower() in (\"sent\", \"sent\")]\n count = len(sent_emails)\n \n if count == 2:\n subjects = [e.get(\"subject\", \"(no subject)\") for e in sent_emails]\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 2 sent emails in mailbox.json. Subjects: {subjects}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 2 sent emails in mailbox.json, but found {count}.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
}
|
|
]
|