101 lines
65 KiB
JSON
101 lines
65 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "2df606af-e182-428c-a0ea-205980c8f1d5",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In `jira_state.json` (external services), there must be a Jira issue with summary 'OFFBOARD-Teodoor-04192026' with Assignee containing: 'Derek Okonkwo' ",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n \n # Try both jira_state.json and jira_data.json\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n jira_path = Path(external_services_path) / \"jira_data.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Neither jira_state.json nor jira_data.json found at {external_services_path}\"}\n \n with open(jira_path, \"r\") as f:\n data = json.load(f)\n \n # Flatten all possible issue structures into a list of issue dicts\n all_issues = []\n \n def collect_issues(obj, depth=0):\n if depth > 10:\n return\n if isinstance(obj, dict):\n # Check if this dict itself looks like an issue (has summary or description or key)\n if any(k in obj for k in [\"summary\", \"description\", \"key\", \"fields\"]):\n all_issues.append(obj)\n # Recurse into values\n for k, v in obj.items():\n if isinstance(v, (dict, list)):\n collect_issues(v, depth + 1)\n elif isinstance(obj, list):\n for item in obj:\n if isinstance(item, (dict, list)):\n collect_issues(item, depth + 1)\n \n collect_issues(data)\n \n # Helper to extract all text from an object\n def get_all_text(obj, depth=0):\n if depth > 10:\n return \"\"\n if isinstance(obj, str):\n return obj + \" \"\n elif isinstance(obj, dict):\n return \" \".join(get_all_text(v, depth + 1) for v in obj.values())\n elif isinstance(obj, list):\n return \" \".join(get_all_text(item, depth + 1) for item in obj)\n else:\n return str(obj) + \" \"\n \n # Helper to get summary from an issue dict\n def get_summary(issue):\n # Direct summary field\n if \"summary\" in issue and isinstance(issue[\"summary\"], str):\n return issue[\"summary\"]\n # Nested in fields\n if \"fields\" in issue and isinstance(issue[\"fields\"], dict):\n if \"summary\" in issue[\"fields\"] and isinstance(issue[\"fields\"][\"summary\"], str):\n return issue[\"fields\"][\"summary\"]\n # Check key field as well\n if \"key\" in issue and isinstance(issue[\"key\"], str):\n return issue[\"key\"]\n return \"\"\n \n # Helper to get assignee from an issue dict\n def get_assignee(issue):\n # Direct assignee field\n if \"assignee\" in issue:\n val = issue[\"assignee\"]\n if isinstance(val, str):\n return val\n elif isinstance(val, dict):\n # Could be {\"name\": ..., \"displayName\": ...}\n for k in [\"displayName\", \"display_name\", \"name\", \"accountId\"]:\n if k in val and isinstance(val[k], str):\n return val[k]\n return get_all_text(val)\n # Nested in fields\n if \"fields\" in issue and isinstance(issue[\"fields\"], dict):\n fields = issue[\"fields\"]\n if \"assignee\" in fields:\n val = fields[\"assignee\"]\n if isinstance(val, str):\n return val\n elif isinstance(val, dict):\n for k in [\"displayName\", \"display_name\", \"name\", \"accountId\"]:\n if k in val and isinstance(val[k], str):\n return val[k]\n return get_all_text(val)\n return \"\"\n \n # Search for the target issue by summary containing 'OFFBOARD-Teodoor-04192026'\n target_summary_lower = \"offboard-teodoor-0419
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "969dc29d-6575-4657-9180-9bac67ea2160",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be a sent email to g.teodoor@crestwood.edu with subject 'Exit Interview Invitation — Teodoor, 10741' containing exactly: \"Hi Gerta,\nAs part of our offboarding process, we would like to offer you the opportunity to participate in an exit interview. This is completely voluntary and confidential. Your feedback helps us improve the employee experience at Crestwood University.\nIf you are interested, please reply with your availability and we will schedule a 30-minute meeting before your last day of 04/19/2026.\nBest regards,\nCrestwood University – Office of Human Resources\nhr@crestwood.edu\"",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n # Try both possible filenames\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n mailbox_path = Path(external_services_path) / \"mailbox_data.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Neither mailbox.json nor mailbox_data.json found at {external_services_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n target_to = \"g.teodoor@crestwood.edu\"\n target_from = \"hr@crestwood.edu\"\n \n # Subject keywords to check\n subject_keywords = [\"teodoor\", \"10741\"]\n \n # The exact expected body from the rubric\n expected_body = (\n \"Hi Gerta,\\n\"\n \"As part of our offboarding process, we would like to offer you the opportunity to participate in an exit interview. \"\n \"This is completely voluntary and confidential. Your feedback helps us improve the employee experience at Crestwood University.\\n\"\n \"If you are interested, please reply with your availability and we will schedule a 30-minute meeting before your last day of 04/19/2026.\\n\"\n \"Best regards,\\n\"\n \"Crestwood University \\u2013 Office of Human Resources\\n\"\n \"hr@crestwood.edu\"\n )\n \n # Normalize whitespace helper\n def normalize(text):\n # Replace various dash characters with a standard em-dash for comparison\n text = text.replace('--', '\\u2013').replace('\\u2014', '\\u2013')\n # Normalize whitespace: replace \\r\\n with \\n\n text = text.replace('\\r\\n', '\\n').replace('\\r', '\\n')\n # Strip each line and rejoin\n lines = [line.strip() for line in text.strip().split('\\n')]\n return '\\n'.join(lines)\n \n normalized_expected = normalize(expected_body)\n \n matching_emails = []\n partial_matches = []\n \n for email in emails:\n to_addr = str(email.get(\"to_addr\", email.get(\"to\", \"\"))).lower()\n from_addr = str(email.get(\"from_addr\", email.get(\"from\", \"\"))).lower()\n subject = str(email.get(\"subject\", \"\")).lower().strip()\n \n to_match = target_to.lower() in to_addr\n from_match = target_from.lower() in from_addr\n \n # Check if subject contains both keywords\n subject_match = all(kw in subject for kw in subject_keywords)\n \n if to_match and from_match and subject_match:\n matching_emails.append(email)\n elif to_match and from_match:\n partial_matches.append(email)\n \n if not matching_emails:\n if partial_matches:\n subjects_found = [str(e.get(\"subject\", \"\")) for e in partial_matches]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found email(s) to {target_to} from {target_from} but subject didn't match. Found subjects: {subjects_found}. Expected subject containing: {subject_keywords}\"}\n \n sent_summaries = []\n for email in emails:\n to_addr = str(email.get(\"to_addr\", email.get(\"to\", \"\")))\n from_addr = str(email.get(\"from_addr\", email.get(\"from\", \"\")))\n subj = str(email.get(\"subject\", \"\"))\n folder = str(email.get(\"folder\", \"\"))\n sent_summaries.append(f\"from={from_addr}, to={to_addr}, subj={subj}, folder={folder}\")\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email found to {target_to} from {target_from} with subject containing {subject_keywords}. All emails in mailbox ({len(emails)} total): {sent_summaries[:10]}\"}\n \n # Check exact match after normalization\n f
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "3793caea-5033-4f41-af82-d14dd6b76da4",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), the email with email_id 5 must be marked as read (is_read=True) and moved to the 'Offboarding' folder.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n # Try both possible filenames: mailbox.json and mailbox_data.json\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n mailbox_path = Path(external_services_path) / \"mailbox_data.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Neither mailbox.json nor mailbox_data.json found at {external_services_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n # The rubric says \"email with email_id 5\". We need to find this email by checking\n # multiple possible key names and multiple possible representations of the id \"2\".\n target_email = None\n for email in emails:\n # Check multiple possible key names for the id\n eid = email.get(\"email_id\", email.get(\"id\", email.get(\"message_id\", \"\")))\n # Accept both integer 2 and string \"2\", as well as any string containing the id\n if eid == 5 or str(eid).strip() == \"5\":\n target_email = email\n break\n \n # If not found by numeric/string id \"2\", also try matching message_id patterns\n # that might correspond to email_id 2 (e.g., the second email in the list, or known message_id)\n if target_email is None:\n # Try matching by known message_id patterns for email_id 2\n known_ids = [\"<supe-fwd-5@crestwood.edu>\"]\n for email in emails:\n msg_id = email.get(\"message_id\", \"\") or email.get(\"email_id\", \"\") or email.get(\"id\", \"\")\n if str(msg_id) in known_ids:\n target_email = email\n break\n \n # If still not found, try the email at index 2 (0-indexed) or index 1 (if 1-indexed list)\n if target_email is None:\n # Try index-based lookup as fallback\n for idx, email in enumerate(emails):\n if idx == 4: # 0-indexed, so index 4 is the 5th email\n # Only use this as candidate if it looks plausible\n pass # Don't auto-select, just note it\n \n if target_email is None:\n all_ids = []\n for email in emails:\n eid = email.get(\"email_id\", email.get(\"id\", email.get(\"message_id\", \"N/A\")))\n all_ids.append(str(eid))\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email with email_id 2 not found in {mailbox_path.name}. Total emails: {len(emails)}. IDs found: {all_ids}\"}\n \n is_read = target_email.get(\"is_read\", False)\n folder = str(target_email.get(\"folder\", \"\")).strip()\n \n issues = []\n if not is_read:\n issues.append(f\"is_read={is_read} (expected True)\")\n if \"offboarding\" not in folder.lower():\n issues.append(f\"folder='{folder}' (expected 'Offboarding' or similar)\")\n \n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email with email_id 2 found but: {'; '.join(issues)}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email with email_id 2 is marked as read (is_read=True) and in folder '{folder}' (matches Offboarding).\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "e637f7e0-0bae-4574-9502-76e8a9ad5e8c",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "In `slack_data.json` (external services), there must be a message in the #hr-offboarding channel containing 'SEPARATION: Gerta Teodoor, 10741, Voluntary Resignation, Last Day: 04/19/2026' and a Jira ticket ID matching the key of the OFFBOARD-Teodoor-04192026 issue.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found at {slack_path}\"}\n \n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n \n # Get Jira ticket key from jira_state.json if available\n jira_key = None\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if jira_path.exists():\n with open(jira_path, \"r\") as f:\n jira_data = json.load(f)\n # Search for the OFFBOARD-Teodoor-04192026 issue or similar\n issues = jira_data.get(\"issues\", jira_data) if isinstance(jira_data, dict) else {}\n if isinstance(issues, dict):\n for key, val in issues.items():\n if \"teodoor\" in key.lower() or (isinstance(val, dict) and \"teodoor\" in str(val).lower()):\n jira_key = key\n break\n elif isinstance(issues, list):\n for issue in issues:\n if isinstance(issue, dict):\n k = issue.get(\"key\", \"\")\n if \"teodoor\" in k.lower():\n jira_key = k\n break\n \n # Also check jira_data.json as fallback\n if jira_key is None:\n jira_path2 = Path(external_services_path) / \"jira_data.json\"\n if jira_path2.exists():\n with open(jira_path2, \"r\") as f:\n jira_data = json.load(f)\n issues = jira_data.get(\"issues\", jira_data) if isinstance(jira_data, dict) else {}\n if isinstance(issues, dict):\n for key, val in issues.items():\n if \"teodoor\" in key.lower() or (isinstance(val, dict) and \"teodoor\" in str(val).lower()):\n jira_key = key\n break\n elif isinstance(issues, list):\n for issue in issues:\n if isinstance(issue, dict):\n k = issue.get(\"key\", \"\")\n if \"teodoor\" in k.lower():\n jira_key = k\n break\n \n if jira_key is None:\n jira_key = \"OFFBOARD-Teodoor-04192026\" # fallback\n \n # Find #hr-offboarding channel\n channels = slack_data.get(\"channels\", {})\n offboarding_channel_id = None\n for ch_id, ch_info in channels.items():\n if isinstance(ch_info, dict) and \"offboarding\" in ch_info.get(\"name\", \"\").lower():\n offboarding_channel_id = ch_id\n break\n \n if offboarding_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No #hr-offboarding channel found in slack_data.json. Channels found: \" + str(list(channels.keys()))}\n \n messages = slack_data.get(\"messages\", {})\n channel_messages = messages.get(offboarding_channel_id, [])\n \n # Required content parts (case-insensitive check)\n required_parts = [\n \"separation\",\n \"gerta teodoor\",\n \"10741\",\n \"voluntary resignation\",\n \"04/19/2026\"\n ]\n \n all_msg_texts = []\n \n for msg in channel_messages:\n if not isinstance(msg, dict):\n continue\n text = str(msg.get(\"text\", \"\"))\n all_msg_texts.append(text[:100])\n text_lower = text.lower()\n \n # Check all required parts present (case-insensitive)\n missing = [part for part in required_parts if part not in text_lower]\n if missing:\n continue\n \n # Check that a Jira ticket ID is mentioned\n has_jira_ref = False\n if jira_key.lower() in
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "1be6180b-f26a-4edd-a4c4-ecfd3866f137",
|
|||
|
|
"sort_order": 4,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), the email with email_id 4 must be marked as read (is_read=True) and moved to the 'General' folder.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n # Try both mailbox.json and mailbox_data.json for flexibility\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n mailbox_path = Path(external_services_path) / \"mailbox_data.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Neither mailbox.json nor mailbox_data.json found at {external_services_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n # Search for email with email_id 4 using multiple possible representations\n for email in emails:\n # Check multiple possible id fields and accept both int and string\n email_id = email.get(\"email_id\", None)\n raw_id = email.get(\"id\", None)\n msg_id = email.get(\"message_id\", None)\n \n # Match email_id 4 in various forms\n matched = False\n for val in [email_id, raw_id]:\n if val is not None:\n try:\n if int(val) == 4:\n matched = True\n break\n except (ValueError, TypeError):\n pass\n if str(val).strip() == \"4\":\n matched = True\n break\n \n # Also check message_id patterns like <mcp-issue-4@...>\n if not matched and msg_id is not None:\n msg_id_str = str(msg_id)\n if \"issue-4@\" in msg_id_str or msg_id_str.strip() == \"4\":\n matched = True\n \n if matched:\n is_read = email.get(\"is_read\", False)\n folder = str(email.get(\"folder\", \"\")).strip()\n \n # Accept truthy values for is_read\n if isinstance(is_read, str):\n is_read_bool = is_read.lower() in (\"true\", \"1\", \"yes\")\n else:\n is_read_bool = bool(is_read)\n \n if not is_read_bool:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email with email_id 4 found but is_read={is_read} (expected True). Folder='{folder}'.\"}\n if folder.lower() != \"general\":\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email with email_id 4 found and is_read=True but folder='{folder}' (expected 'General')\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email with email_id 4 is marked as read (is_read=True) and in folder='{folder}' (matches 'General').\"}\n \n # Provide detailed debug info on what emails were found\n found_ids = []\n for email in emails:\n eid = email.get(\"email_id\", email.get(\"id\", email.get(\"message_id\", \"unknown\")))\n found_ids.append(str(eid))\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email with email_id 4 not found in {mailbox_path.name}. Found {len(emails)} emails with ids: {found_ids[:20]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "c30ba66b-44c4-4ca6-91ff-df24ba186ee1",
|
|||
|
|
"sort_order": 5,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), there must be 1 email in_reply_to the email with message ID '<mcp-issue-4@crestwood.edu>', with subject that contains 'Inquiry: Pay discrepancy and coworker conduct', containing: \"Thank you for contacting us. Your message has been forwarded to our Employee Relations Specialist, who will follow up with you directly. Please allow 2 business days for initial contact.\", \"Thank you for reaching out. For all payroll and compensation questions, please contact our Payroll Liaison,\nCalvin Briggs, directly at c.briggs@crestwood.edu. Your message has been forwarded to him.\", \"Hi Ardisj,\", and \"Best regards,\nCrestwood University – Office of Human Resources\nhr@crestwood.edu\"",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n # Try both possible filenames\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n mailbox_path = Path(external_services_path) / \"mailbox_data.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Neither mailbox.json nor mailbox_data.json found at {external_services_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n reply_to_id = \"mcp-issue-4@crestwood.edu\"\n target_subject_fragment = \"inquiry: pay discrepancy and coworker conduct\"\n \n def normalize(text):\n \"\"\"Normalize whitespace and common character variations for loose matching.\"\"\"\n text = str(text)\n # Normalize various dash/hyphen characters to standard hyphen\n text = re.sub(r'[\\u2013\\u2014\\u2015\\u2212]', '-', text)\n # Normalize whitespace (newlines, tabs, multiple spaces) to single space\n text = re.sub(r'\\s+', ' ', text).strip().lower()\n return text\n \n def loose_contains(body, snippet):\n return normalize(snippet) in normalize(body)\n \n # Find matching reply emails: in_reply_to contains mcp-issue-4@crestwood.edu, subject contains target\n matching_emails = []\n for email in emails:\n in_reply_to = str(email.get(\"in_reply_to\", \"\") or \"\").strip()\n references = str(email.get(\"references\", \"\") or \"\").strip()\n subject = str(email.get(\"subject\", \"\") or \"\").strip().lower()\n \n # Check reply match\n reply_match = (\"mcp-issue-4@crestwood.edu\" in in_reply_to or\n \"mcp-issue-4\" in in_reply_to or\n \"mcp-issue-4\" in references)\n \n # Check subject match\n subject_clean = re.sub(r'^(re:\\s*)+', '', subject).strip()\n subject_match = (target_subject_fragment in subject_clean or\n target_subject_fragment in subject or\n \"pay discrepancy\" in subject)\n \n if reply_match and subject_match:\n matching_emails.append(email)\n \n if not matching_emails:\n # Debug info\n debug_info = []\n for email in emails:\n in_reply_to = str(email.get(\"in_reply_to\", \"\") or \"\")\n if \"mcp-issue-4\" in in_reply_to:\n debug_info.append(f\"subject={email.get('subject')}, in_reply_to={in_reply_to}\")\n \n all_sent = []\n for email in emails:\n folder = str(email.get(\"folder\", \"\")).lower()\n if folder == \"sent\" or \"sent\" in folder:\n all_sent.append(f\"to={email.get('to_addr', email.get('to'))}, subject={email.get('subject')}, in_reply_to={email.get('in_reply_to')}\")\n sent_str = \"; \".join(all_sent[:10]) if all_sent else \"No sent emails found\"\n debug_str = \"; \".join(debug_info[:5]) if debug_info else \"No emails with mcp-issue-4 in_reply_to\"\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No emails found replying to mcp-issue-4 with matching subject. Debug: {debug_str}. Sent emails: {sent_str}. Total emails: {len(emails)}\"}\n \n # The rubric says there must be 1 email containing ALL four snippets.\n # Define the four required content snippets:\n snippet_er = \"Thank you for contacting us. Your message has been forwarded to our Employee Relations Specialist, who will follow up with you directly. Please allow 2 business days for initial contact.\"\n snippet_payroll = \"Thank you for reaching out. For all payroll and compensation questions, please contact our Payroll Liaison, Calvin Briggs
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "97829e3a-6355-46f9-8df4-0b91c56152d5",
|
|||
|
|
"sort_order": 6,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), email_id 4 must be forwarded to m.voss@crestwood.edu with a subject line that contains 'Inquiry: Pay discrepancy and coworker conduct'.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n \n # Try multiple possible filenames\n mailbox_path = None\n for fname in [\"mailbox.json\", \"mailbox_data.json\"]:\n candidate = Path(external_services_path) / fname\n if candidate.exists():\n mailbox_path = candidate\n break\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Neither mailbox.json nor mailbox_data.json found at {external_services_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", []) if isinstance(data, dict) else data if isinstance(data, list) else []\n \n target_to = \"m.voss@crestwood.edu\"\n subject_key_phrase = \"inquiry: pay discrepancy and coworker conduct\"\n \n # We look for any email sent to m.voss@crestwood.edu whose subject contains\n # the key phrase (case-insensitive). This represents forwarding email_id 4.\n # We are lenient: subject can have prefixes like Fwd:, FW:, Re: etc.\n \n all_candidates = []\n matched = []\n \n for email in emails:\n to_addr = str(email.get(\"to_addr\", email.get(\"to\", \"\"))).lower()\n \n to_match = target_to.lower() in to_addr\n \n if not to_match:\n continue\n \n all_candidates.append(email)\n \n subject = str(email.get(\"subject\", \"\")).strip()\n subject_lower = subject.lower()\n \n # Check if the subject contains the key phrase (with or without Fwd:/Re: prefixes)\n if subject_key_phrase in subject_lower:\n matched.append(email)\n \n if matched:\n email = matched[0]\n subject = str(email.get(\"subject\", \"\"))\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found email forwarded to {target_to} with subject '{subject}' containing 'Inquiry: Pay discrepancy and coworker conduct'.\"}\n \n if all_candidates:\n subjects = [str(e.get(\"subject\", \"\")) for e in all_candidates]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found email(s) to {target_to}, but none with subject containing 'Inquiry: Pay discrepancy and coworker conduct'. Found subjects: {subjects}\"}\n \n all_info = [(str(e.get(\"to_addr\", e.get(\"to\", \"\"))), str(e.get(\"subject\", \"\"))) for e in emails]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email found to {target_to}. Total emails in mailbox: {len(emails)}. Sample entries (to, subject): {all_info[:10]}\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "48ec34d4-2af0-46ba-ba82-86ae4271d1c1",
|
|||
|
|
"sort_order": 7,
|
|||
|
|
"rubric_text": "In `mailbox.json` (external services), email_id 4 must be forwarded to c.briggs@crestwood.edu with a subject line that contains 'Inquiry: Pay discrepancy and coworker conduct'.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json or mailbox_data.json\"}\n \n # Try both possible filenames\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n mailbox_path = Path(external_services_path) / \"mailbox_data.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Neither mailbox.json nor mailbox_data.json found at {external_services_path}\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n target_to = \"c.briggs@crestwood.edu\"\n target_subject_phrase = \"inquiry: pay discrepancy and coworker conduct\"\n \n matching_emails = []\n for email in emails:\n to_addr = str(email.get(\"to_addr\", email.get(\"to\", \"\"))).lower()\n subject = str(email.get(\"subject\", \"\")).lower().strip()\n \n # Check to address\n if target_to.lower() not in to_addr:\n continue\n \n # Check subject contains the target phrase (loose: allow Fwd:, Re:, etc.)\n if target_subject_phrase not in subject:\n continue\n \n matching_emails.append(email)\n \n if not matching_emails:\n # Provide diagnostic info\n all_info = [(str(e.get(\"to_addr\", e.get(\"to\", \"\"))), str(e.get(\"subject\", \"\"))) for e in emails]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No email found to {target_to} with subject containing '{target_subject_phrase}'. Found emails: {all_info[:15]}\"}\n \n # We found at least one matching email\n email = matching_emails[0]\n subject = str(email.get(\"subject\", \"\"))\n from_addr = str(email.get(\"from_addr\", email.get(\"from\", \"\")))\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found email forwarded to {target_to} with subject '{subject}' from '{from_addr}'. Subject contains required phrase.\"\n }\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "7383fa5a-7274-40b2-badf-95de87b1a865",
|
|||
|
|
"sort_order": 8,
|
|||
|
|
"rubric_text": "In the external calendar service (`calendar_data.json`), there must NOT be any calendar event for an exit interview for Teodoor.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n # No calendar service available - can't verify absence, pass by default\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"external_services_path is None; no calendar to check - assuming no exit interview scheduled\"}\n \n calendar_path = Path(external_services_path) / \"calendar_data.json\"\n if not calendar_path.exists():\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"calendar_data.json not found - no exit interview calendar event can exist\"}\n \n with open(calendar_path, \"r\") as f:\n data = json.load(f)\n \n events = data.get(\"events\", {})\n \n # Check for any event related to Teodoor exit interview\n suspicious_events = []\n for event_id, event in events.items():\n if not isinstance(event, dict):\n continue\n summary = str(event.get(\"summary\", \"\")).lower()\n description = str(event.get(\"description\", \"\")).lower()\n combined = summary + \" \" + description\n \n if \"teodoor\" in combined and (\"exit\" in combined or \"interview\" in combined):\n suspicious_events.append({\"id\": event_id, \"summary\": event.get(\"summary\", \"\")})\n \n if suspicious_events:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found calendar exit interview event(s) for Teodoor that should NOT exist: {suspicious_events}\"}\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No calendar exit interview event for Teodoor found (as required)\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776476136260",
|
|||
|
|
"sort_order": 9,
|
|||
|
|
"rubric_text": "In `jira_state.json`, there must exist a ticket with title `BENEFITS-Parke-04092026` (case-insensitive) with Assignee contains either `Calvin Briggs` or `Patricia Huang`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef verify(workspace_path, external_services_path=None):\n # --- Locate jira_state.json ---\n jira_path = None\n candidates = []\n if external_services_path:\n candidates.append(Path(external_services_path) / \"jira_state.json\")\n if workspace_path:\n candidates.append(Path(workspace_path) / \"jira_state.json\")\n for candidate in candidates:\n if candidate.exists():\n jira_path = candidate\n break\n\n if jira_path is None:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"Could not find jira_state.json in external_services_path or workspace_path.\",\n }\n\n try:\n with open(jira_path, \"r\") as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse jira_state.json: {e}\"}\n\n # --- Target configuration ---\n target_title_lower = \"benefits-parke-04092026\"\n valid_assignees_lower = [\"calvin briggs\", \"patricia huang\"]\n\n # --- Collect all ticket dicts regardless of container shape ---\n tickets = []\n if isinstance(jira_data, list):\n tickets = [t for t in jira_data if isinstance(t, dict)]\n elif isinstance(jira_data, dict):\n if isinstance(jira_data.get(\"tickets\"), list):\n tickets.extend(t for t in jira_data[\"tickets\"] if isinstance(t, dict))\n if isinstance(jira_data.get(\"issues\"), list):\n tickets.extend(t for t in jira_data[\"issues\"] if isinstance(t, dict))\n if isinstance(jira_data.get(\"issues\"), dict):\n for k, v in jira_data[\"issues\"].items():\n if isinstance(v, dict):\n if \"key\" not in v:\n v = {**v, \"key\": k}\n tickets.append(v)\n # Fallback: top-level dict mapping ticket_id -> ticket_obj\n if not tickets:\n for k, v in jira_data.items():\n if isinstance(v, dict) and any(f in v for f in (\"summary\", \"fields\", \"title\", \"assignee\", \"key\", \"description\")):\n if \"key\" not in v:\n v = {**v, \"key\": k}\n tickets.append(v)\n\n # --- Helpers ---\n def pick(ticket, *names):\n for source in (ticket, ticket.get(\"fields\") if isinstance(ticket.get(\"fields\"), dict) else None):\n if not isinstance(source, dict):\n continue\n for n in names:\n v = source.get(n)\n if v:\n return v\n return None\n\n def get_title(ticket):\n v = pick(ticket, \"summary\", \"Summary\", \"title\", \"Title\", \"name\", \"Name\")\n return str(v).strip() if v is not None else \"\"\n\n def get_key(ticket):\n v = pick(ticket, \"key\", \"Key\", \"id\", \"ID\", \"Id\")\n return str(v).strip() if v is not None else \"\"\n\n def get_assignee_name(ticket):\n v = pick(ticket, \"assignee\", \"Assignee\", \"assigned_to\", \"AssignedTo\")\n if v is None:\n return \"\"\n if isinstance(v, dict):\n for field in (\"displayName\", \"display_name\", \"name\", \"real_name\", \"username\", \"fullName\", \"full_name\"):\n if v.get(field):\n return str(v[field])\n return \"\"\n return str(v)\n\n # --- Normalize for matching ---\n def normalize(s):\n \"\"\"Lowercase and replace common separators with '-' for flexible matching.\"\"\"\n return re.sub(r'[-_\\s]+', '-', s.strip().lower())\n\n target_normalized = normalize(target_title_lower)\n\n found_summaries = []\n matched_ticket = None\n for ticket in tickets:\n title = get_title(ticket)\n key = get_key(ticket)\n found_summaries.append(f\"{key} :: {title}\")\n\n # Check if title or key matches (case-insensitive, separator-flexib
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776476266978",
|
|||
|
|
"sort_order": 10,
|
|||
|
|
"rubric_text": "In `jira_state.json`, there must exist exactly 21 tickets.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for jira_state.json\n jira_path = None\n \n # Try external services path first\n if external_services_path:\n candidate = Path(external_services_path) / \"jira_state.json\"\n if candidate.exists():\n jira_path = candidate\n \n # Also try workspace path\n if jira_path is None:\n candidate = Path(workspace_path) / \"jira_state.json\"\n if candidate.exists():\n jira_path = candidate\n \n if jira_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find jira_state.json in either workspace or external services path.\"}\n \n try:\n with open(jira_path, 'r') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse jira_state.json: {e}\"}\n \n # Count tickets - handle various possible structures\n ticket_count = 0\n \n if isinstance(jira_data, list):\n ticket_count = len(jira_data)\n elif isinstance(jira_data, dict):\n # Could be {\"tickets\": [...]} or {\"issues\": [...]} or the dict itself maps ticket IDs to ticket data\n if \"tickets\" in jira_data:\n tickets = jira_data[\"tickets\"]\n if isinstance(tickets, list):\n ticket_count = len(tickets)\n elif isinstance(tickets, dict):\n ticket_count = len(tickets)\n elif \"issues\" in jira_data:\n issues = jira_data[\"issues\"]\n if isinstance(issues, list):\n ticket_count = len(issues)\n elif isinstance(issues, dict):\n ticket_count = len(issues)\n else:\n # Maybe the top-level dict keys are ticket IDs\n # Check if values look like ticket objects (dicts with fields)\n all_values_are_dicts = all(isinstance(v, dict) for v in jira_data.values()) if jira_data else False\n if all_values_are_dicts:\n ticket_count = len(jira_data)\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Unrecognized jira_state.json structure. Top-level keys: {list(jira_data.keys())[:10]}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Unexpected jira_state.json format: top-level type is {type(jira_data).__name__}\"}\n \n if ticket_count == 21:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 21 tickets in jira_state.json as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 21 tickets in jira_state.json, but found {ticket_count}.\"}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776477719669",
|
|||
|
|
"sort_order": 11,
|
|||
|
|
"rubric_text": "In `mailbox.json`, there must be a sent email to `j.park@crestwood.edu` with subject line `Access Revocation Request — Teodoor, 10741` that contains the following body text:\n\n\"Hi Jenna,\n\nPlease revoke all system access for the following employee:\n\nName: Gerta Teodoor\nEmployee ID: 10741\nLast Day of Work: 04/19/2026.\nTiming: End of business on last day\n\nThis includes email, VPN, building access, and all university systems. Please confirm when complete.\n\nBest regards,\nCrestwood University – Office of Human Resources\nhr@crestwood.edu\"",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json for sent emails.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n \n with open(mailbox_path, \"r\", encoding=\"utf-8\") as f:\n mailbox_data = json.load(f)\n \n emails = mailbox_data.get(\"emails\", [])\n \n # Find sent emails to j.park@crestwood.edu with matching subject\n target_to = \"j.park@crestwood.edu\"\n target_subject = \"Access Revocation Request — Teodoor, 10741\"\n \n # Also accept common dash variants\n def normalize_dashes(s):\n # Replace em-dash, en-dash, and various Unicode dashes with a standard dash for comparison\n return re.sub(r'[\\u2014\\u2013\\u2012\\u2015\\u2010\\u002D\\uFE58\\uFE63\\uFF0D—–-]', '-', s)\n \n target_subject_normalized = normalize_dashes(target_subject.lower().strip())\n \n matching_emails = []\n for email in emails:\n folder = email.get(\"folder\", \"\").lower()\n to_addr = email.get(\"to_addr\", \"\").lower().strip()\n subject = email.get(\"subject\", \"\").strip()\n subject_normalized = normalize_dashes(subject.lower().strip())\n \n if folder == \"sent\" and target_to.lower() in to_addr and target_subject_normalized == subject_normalized:\n matching_emails.append(email)\n \n if not matching_emails:\n # Provide more debugging info\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n sent_subjects = [e.get(\"subject\", \"\") for e in sent_emails]\n sent_tos = [e.get(\"to_addr\", \"\") for e in sent_emails]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No sent email found to '{target_to}' with subject '{target_subject}'. Found {len(sent_emails)} sent emails. Subjects: {sent_subjects}. To addresses: {sent_tos}\"\n }\n \n # Now check the body content of matching emails\n required_fragments = [\n \"Hi Jenna\",\n \"revoke all system access\",\n \"Gerta Teodoor\",\n \"10741\",\n \"04/19/2026\",\n \"End of business on last day\",\n \"email\",\n \"VPN\",\n \"building access\",\n \"university systems\",\n \"confirm when complete\",\n \"Crestwood University\",\n \"Office of Human Resources\",\n \"hr@crestwood.edu\"\n ]\n \n for email in matching_emails:\n body = email.get(\"body_text\", \"\")\n # Normalize whitespace for comparison\n body_normalized = re.sub(r'\\s+', ' ', body).strip().lower()\n \n missing = []\n for frag in required_fragments:\n frag_normalized = re.sub(r'\\s+', ' ', frag).strip().lower()\n if frag_normalized not in body_normalized:\n missing.append(frag)\n \n if not missing:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found sent email to '{target_to}' with correct subject and all required body content present.\"\n }\n else:\n # Partial match - check how many are present\n found_count = len(required_fragments) - len(missing)\n total = len(required_fragments)\n # Be generous: if only 1-2 minor fragments missing, still pass\n if len(missing) <= 1:\n return {\n \"pass\": True,\n \"score\": 0.9,\n \"feedback\": f\"Found sent email with nearly all required content ({found_count}/{total}). Minor missing fragments: {missing}\"\n
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776478647283",
|
|||
|
|
"sort_order": 12,
|
|||
|
|
"rubric_text": "In mailbox.json, email email_id 4 must be starred (flagged as important) and in folder “General”.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Locate mailbox.json\n mailbox_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n if not mailbox_path:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n\n if not mailbox_path:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": \"Could not find mailbox.json in either external_services_path or workspace_path.\"}\n\n try:\n with open(mailbox_path, 'r') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n emails = data.get(\"emails\", [])\n target_id = 4\n\n target_email = None\n for email in emails:\n email_id_raw = email.get(\"email_id\", email.get(\"id\"))\n if email_id_raw is None:\n continue\n try:\n if int(email_id_raw) == target_id:\n target_email = email\n break\n except (ValueError, TypeError):\n continue\n\n if target_email is None:\n return {\"pass\": False, \"score\": 0.0,\n \"feedback\": f\"Email with email_id {target_id} was not found in mailbox.json.\"}\n\n # --- Check 1: starred / important ---\n is_starred = target_email.get(\"is_starred\", False)\n is_important = target_email.get(\"is_important\", False)\n starred = target_email.get(\"starred\", False)\n important = target_email.get(\"important\", False)\n labels = target_email.get(\"labels\", [])\n\n flagged = bool(is_starred or is_important or starred or important)\n if not flagged and isinstance(labels, list):\n for label in labels:\n label_lower = str(label).lower()\n if \"important\" in label_lower or \"starred\" in label_lower:\n flagged = True\n break\n\n # --- Check 2: folder is \"General\" ---\n folder = target_email.get(\"folder\", None)\n folder_ok = isinstance(folder, str) and folder.strip().lower() == \"general\"\n\n problems = []\n if not flagged:\n problems.append(f\"email_id {target_id} is NOT flagged as important/starred \"\n f\"(is_starred={is_starred}, is_important={is_important}, \"\n f\"starred={starred}, important={important}, labels={labels})\")\n if not folder_ok:\n problems.append(f\"email_id {target_id} is in folder {folder!r}, expected 'General'\")\n\n if problems:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"FAIL: \" + \"; \".join(problems)}\n\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": f\"PASS: email_id {target_id} is flagged as important/starred and is in folder 'General'.\"}\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "rubric_1776484300062",
|
|||
|
|
"sort_order": 13,
|
|||
|
|
"rubric_text": "In `jira_state.json`, there must not be a ticket named `ER-Parke-04092026`.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check jira_state.json in external services first, then workspace\n jira_path = None\n \n # Try external services path\n if external_services_path:\n candidate = Path(external_services_path) / \"jira_state.json\"\n if candidate.exists():\n jira_path = candidate\n \n # Try workspace path\n if jira_path is None:\n candidate = Path(workspace_path) / \"jira_state.json\"\n if candidate.exists():\n jira_path = candidate\n \n if jira_path is None:\n # If jira_state.json doesn't exist anywhere, then the ticket certainly doesn't exist -> pass\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"jira_state.json not found in workspace or external services. The forbidden ticket cannot exist, so the condition is satisfied.\"}\n \n try:\n with open(jira_path, 'r') as f:\n jira_data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse jira_state.json at {jira_path}: {e}\"}\n \n forbidden_ticket = \"ER-Parke-04092026\"\n \n # Search through jira_data for any ticket with this name/key\n # jira_state.json can have various structures; search broadly\n \n def search_for_ticket(obj, path=\"\"):\n \"\"\"Recursively search for the forbidden ticket name in the JSON structure.\"\"\"\n findings = []\n if isinstance(obj, dict):\n # Check if this dict has the ticket as a key\n for key, value in obj.items():\n if key.strip().lower() == forbidden_ticket.lower():\n findings.append(f\"Found ticket as key at {path}.{key}\")\n # Check if value fields contain the ticket name\n if isinstance(value, str) and value.strip().lower() == forbidden_ticket.lower():\n findings.append(f\"Found ticket as value at {path}.{key} = {value}\")\n # Also check common field names like 'name', 'title', 'key', 'id', 'ticket_id', 'summary'\n if key.lower() in ('name', 'title', 'key', 'id', 'ticket_id', 'summary', 'ticket_name', 'ticket_key'):\n if isinstance(value, str) and value.strip().lower() == forbidden_ticket.lower():\n findings.append(f\"Found ticket in field '{key}' at {path}: {value}\")\n findings.extend(search_for_ticket(value, f\"{path}.{key}\"))\n elif isinstance(obj, list):\n for i, item in enumerate(obj):\n findings.extend(search_for_ticket(item, f\"{path}[{i}]\"))\n return findings\n \n findings = search_for_ticket(jira_data)\n \n if findings:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Found forbidden ticket 'ER-Parke-04092026' in jira_state.json. Details: {'; '.join(findings)}\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: No ticket named 'ER-Parke-04092026' found in jira_state.json at {jira_path}.\"\n }\n",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
}
|
|||
|
|
]
|