Add Handbook.md benchmark tasks
This commit is contained in:
DerekSurge
@@ -0,0 +1,107 @@
|
||||
[
|
||||
{
|
||||
"id": "3dc27ca8-d8b7-4cf5-9e1e-a047bd7ac982",
|
||||
"sort_order": 0,
|
||||
"rubric_text": "In `mailbox.json`, the email with subject 'Delivery Waiting!!!!!!!' from t.downes@texpal.com must be forwarded to carlos.mendoza@prairiestar.com",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n # Look for a forwarded email about 'Delivery Waiting!!!!!!!' to carlos.mendoza\n for e in sent_emails:\n subject = e.get(\"subject\", \"\")\n to_addr = e.get(\"to_addr\", \"\") or \"\"\n cc_addr = e.get(\"cc_addr\", \"\") or \"\"\n all_recipients = (to_addr + \" \" + cc_addr).lower()\n if \"delivery waiting\" in subject.lower() and \"carlos.mendoza@prairiestar.com\" in all_recipients:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found forwarded email with subject '{subject}' to carlos.mendoza@prairiestar.com\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No forwarded email with subject containing 'Delivery Waiting' to carlos.mendoza@prairiestar.com found in Sent folder\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "25656f51-ffe3-4255-9b9a-f5efab012384",
|
||||
"sort_order": 1,
|
||||
"rubric_text": "In `slack_data.json`, channel `#dm-c.mendoza` must contain a message with exact content 'OUT OF SCOPE: Delivery Waiting!!!!!!! -- please advise' (dash separator variations acceptable)",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n target_channel_id = None\n for cid, cinfo in channels.items():\n name = cinfo.get(\"name\", \"\").lower().replace(\".\", \"-\").replace(\"_\", \"-\")\n if \"dm\" in name and \"mendoza\" in name:\n target_channel_id = cid\n break\n if target_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #dm-c.mendoza channel in slack_data.json\"}\n channel_msgs = messages.get(target_channel_id, [])\n # Strict: anchored full-string match, no additional content allowed.\n # Rubric explicitly allows dash-separator variations (- / -- / – / —); everything else is literal.\n pattern = r'^out of scope:\\s*delivery waiting!{7}\\s*[-\\u2013\\u2014]+\\s*please advise$'\n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\").strip()\n if re.match(pattern, text, re.IGNORECASE):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found message in #dm-c.mendoza: '{text}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No exact-match 'OUT OF SCOPE: Delivery Waiting!!!!!!! -- please advise' message found in #dm-c.mendoza. Messages: {[m.get('text','') for m in channel_msgs]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "efced52e-df02-424a-840a-6bd7ded35a82",
|
||||
"sort_order": 2,
|
||||
"rubric_text": "In `slack_data.json`, channel `#dm-c.mendoza` must contain a message with exact content 'URGENT: Pest Contamination – TexPal shipment received 2026-04-01'",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n target_channel_id = None\n for cid, cinfo in channels.items():\n name = cinfo.get(\"name\", \"\").lower().replace(\".\", \"-\").replace(\"_\", \"-\")\n if \"dm\" in name and \"mendoza\" in name:\n target_channel_id = cid\n break\n if target_channel_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find #dm-c.mendoza channel in slack_data.json\"}\n channel_msgs = messages.get(target_channel_id, [])\n # Strict: anchored full-string match, no additional content allowed.\n # Accepts \"TexPal\" or \"TexPal Logistics\"; accepts -, –, — as the separator.\n pattern = r'^urgent:\\s*pest\\s+contamination\\s*[-\\u2013\\u2014]\\s*texpal(\\s+logistics)?\\s+shipment\\s+received\\s+2026-04-01$'\n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\").strip()\n if re.match(pattern, text, re.IGNORECASE):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found Pest Contamination URGENT message in #dm-c.mendoza: '{text}'\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No exact-match URGENT Pest Contamination message found in #dm-c.mendoza. Messages: {[m.get('text','') for m in channel_msgs]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "f64ad1ee-0161-4f03-8e07-184943e71ed5",
|
||||
"sort_order": 3,
|
||||
"rubric_text": "In `mailbox.json`, the email with subject 'Pest Contamination' from sarah.kim@prairiestar.com must be forwarded with both daniel.reyes@prairiestar.com and carlos.mendoza@prairiestar.com as recipients",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n subject = e.get(\"subject\", \"\")\n to_addr = e.get(\"to_addr\", \"\") or \"\"\n cc_addr = e.get(\"cc_addr\", \"\") or \"\"\n all_recipients = (to_addr + \" \" + cc_addr).lower()\n if \"pest contamination\" in subject.lower():\n has_daniel = \"daniel.reyes@prairiestar.com\" in all_recipients\n has_carlos = \"carlos.mendoza@prairiestar.com\" in all_recipients\n if has_daniel and has_carlos:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found forwarded Pest Contamination email with both daniel.reyes and carlos.mendoza as recipients\"}\n else:\n missing = []\n if not has_daniel: missing.append(\"daniel.reyes@prairiestar.com\")\n if not has_carlos: missing.append(\"carlos.mendoza@prairiestar.com\")\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found Pest Contamination forward but missing recipients: {missing}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No forwarded email with subject 'Pest Contamination' found in Sent folder\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "ef1969dc-8986-4758-a00f-d5c4796c8a83",
|
||||
"sort_order": 4,
|
||||
"rubric_text": "In `slack_data.json`, channels `#cold-chain-alerts` and `#dc-escalations-houston` must each contain a message with content 'URGENT: issue with TexPal for shipment received on 2026-04-01' (or with 'TexPal Logistics' instead of 'TexPal'), with no additional content",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n def find_channel_id(target_name):\n for cid, cinfo in channels.items():\n name = cinfo.get(\"name\", \"\").lower().replace(\"_\", \"-\")\n if target_name.lower().replace(\"_\", \"-\") in name:\n return cid\n return None\n\n def msg_matches(text):\n # Accept TexPal or TexPal Logistics, case insensitive\n pattern = r'^urgent:\\s*issue with texpal(\\s+logistics)?\\s+for\\s+shipment\\s+received\\s+on\\s+2026-04-01$'\n return bool(re.match(pattern, text.strip(), re.IGNORECASE))\n\n results = {}\n for channel_name in [\"cold-chain-alerts\", \"dc-escalations-houston\"]:\n cid = find_channel_id(channel_name)\n if cid is None:\n results[channel_name] = f\"Channel #{channel_name} not found\"\n continue\n channel_msgs = messages.get(cid, [])\n found = False\n for msg in channel_msgs:\n text = msg.get(\"text\", \"\")\n if msg_matches(text):\n found = True\n break\n if not found:\n texts = [m.get(\"text\", \"\") for m in channel_msgs]\n results[channel_name] = f\"No matching URGENT TexPal message found. Messages: {texts}\"\n else:\n results[channel_name] = \"OK\"\n\n all_pass = all(v == \"OK\" for v in results.values())\n if all_pass:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found URGENT TexPal shipment messages in both #cold-chain-alerts and #dc-escalations-houston\"}\n else:\n feedback = \"; \".join(f\"{k}: {v}\" for k, v in results.items())\n return {\"pass\": False, \"score\": 0.0, \"feedback\": feedback}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "a572ce7a-41ec-45ad-806c-60fe72c26842",
|
||||
"sort_order": 5,
|
||||
"rubric_text": "In `inbound_receiving_log.xlsx`, cell H2 must be 'Class 4' and cell I2 must be 'Rejected – Class 4 Contamination'",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n files = list(Path(workspace_path).glob(\"*inbound_receiving_log*\"))\n if not files:\n files = list(Path(workspace_path).glob(\"*.xlsx\"))\n files = [f for f in files if \"inbound\" in f.name.lower() or \"receiving\" in f.name.lower()]\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No inbound_receiving_log.xlsx found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n h2 = ws[\"H2\"].value\n i2 = ws[\"I2\"].value\n h2_str = str(h2).strip() if h2 is not None else \"\"\n i2_str = str(i2).strip() if i2 is not None else \"\"\n h2_ok = h2_str.lower() == \"class 4\"\n # Accept dash variations: – — -\n i2_normalized = re.sub(r'[\\u2013\\u2014\\-]', '-', i2_str).lower()\n i2_ok = \"rejected\" in i2_normalized and \"class 4\" in i2_normalized and \"contamination\" in i2_normalized\n if h2_ok and i2_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"H2='{h2_str}', I2='{i2_str}' - both correct\"}\n feedback_parts = []\n if not h2_ok:\n feedback_parts.append(f\"H2 expected 'Class 4' but got '{h2_str}'\")\n if not i2_ok:\n feedback_parts.append(f\"I2 expected 'Rejected – Class 4 Contamination' but got '{i2_str}'\")\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(feedback_parts)}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "4536b9b9-bea3-4876-a146-cf10b5529d90",
|
||||
"sort_order": 6,
|
||||
"rubric_text": "In `mailbox.json`, there must be a sent email to t.downes@texpal.com with daniel.reyes@prairiestar.com and marcus.liu@prairiestar.com CC'd, subject 'PRIORITY – Shipment Rejection – Manifest TP56734 – 2026-04-01', with body containing manifest TP56734, date 2026-04-01, reason Class 4 Contamination, and signed Prairie Star Creamery",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n to_addr = e.get(\"to_addr\", \"\") or \"\"\n cc_addr = e.get(\"cc_addr\", \"\") or \"\"\n subject = e.get(\"subject\", \"\") or \"\"\n body = e.get(\"body_text\", \"\") or \"\"\n if \"t.downes@texpal.com\" not in to_addr.lower():\n continue\n # Check subject - normalize dashes\n subj_norm = re.sub(r'[\\u2013\\u2014]', '-', subject)\n if not re.search(r'priority.*shipment rejection.*(manifest\\s+|po\\s*)?tp56734.*2026-04-01', subj_norm, re.IGNORECASE):\n continue\n # Check CC\n cc_lower = cc_addr.lower()\n has_daniel = \"daniel.reyes@prairiestar.com\" in cc_lower\n has_marcus = \"marcus.liu@prairiestar.com\" in cc_lower\n # Check body content\n body_lower = body.lower()\n has_manifest = \"tp56734\" in body_lower\n has_date = \"2026-04-01\" in body\n has_class4 = \"class 4\" in body_lower\n has_contamination = \"contamination\" in body_lower\n has_prairie = \"prairie star\" in body_lower\n checks = {\"daniel CC\": has_daniel, \"marcus CC\": has_marcus, \"manifest TP56734 in body\": has_manifest,\n \"date 2026-04-01 in body\": has_date, \"Class 4 in body\": has_class4,\n \"Contamination in body\": has_contamination, \"Prairie Star in body\": has_prairie}\n all_ok = all(checks.values())\n if all_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found correct rejection email to t.downes@texpal.com with subject '{subject}'\"}\n else:\n failed = [k for k, v in checks.items() if not v]\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found rejection email but missing: {failed}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email to t.downes@texpal.com with subject matching 'PRIORITY – Shipment Rejection – Manifest TP56734 – 2026-04-01' found\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "e02229cd-4ddd-4fb6-8eab-17ba07579b31",
|
||||
"sort_order": 7,
|
||||
"rubric_text": "File `IR-2026-04-01-Contamination.pdf` must exist in the workspace, must be the only file with 'IR' prefix, and must contain Incident Date: 2026-04-01, Incident Category: Contamination, Priority Level: P1",
|
||||
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n # Find all files starting with IR (case insensitive)\n ir_files = [f for f in workspace.glob(\"*\") if f.is_file() and f.name.upper().startswith(\"IR\")]\n if len(ir_files) == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No file with 'IR' prefix found in workspace\"}\n if len(ir_files) > 1:\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"Multiple IR-prefixed files found: {[f.name for f in ir_files]}. Expected exactly one.\"}\n ir_file = ir_files[0]\n if \"2026-04-01\" not in ir_file.name or \"contamination\" not in ir_file.name.lower():\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"IR file found as '{ir_file.name}' but expected name to contain '2026-04-01' and 'Contamination'\"}\n if ir_file.suffix.lower() != \".pdf\":\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"IR file '{ir_file.name}' is not a PDF\"}\n # Check PDF content\n try:\n import pdfplumber\n with pdfplumber.open(ir_file) as pdf:\n text = \"\"\n for page in pdf.pages:\n text += (page.extract_text() or \"\")\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.3, \"feedback\": f\"Could not read PDF '{ir_file.name}': {ex}\"}\n text_lower = text.lower()\n checks = {\n \"Incident Date 2026-04-01\": \"2026-04-01\" in text,\n \"Incident Category Contamination\": \"contamination\" in text_lower,\n \"Priority Level P1\": bool(re.search(r'p1|priority.*1|1.*priority', text_lower))\n }\n all_ok = all(checks.values())\n if all_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"IR file '{ir_file.name}' exists with correct content: date 2026-04-01, Contamination, P1\"}\n failed = [k for k, v in checks.items() if not v]\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"IR file '{ir_file.name}' exists but missing content: {failed}. Extracted text snippet: {text[:500]}\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "7b25afa2-8e11-49bd-96d2-d9ea3dcdd1c9",
|
||||
"sort_order": 8,
|
||||
"rubric_text": "In `mailbox.json`, there must be a sent email to carlos.mendoza@prairiestar.com with subject 'IR-2026-04-01-Contamination.pdf for review' or 'IR-2026-04-01-Contamination for review', a blank body, and `IR-2026-04-01-Contamination.pdf` attached",
|
||||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n to_addr = (e.get(\"to_addr\", \"\") or \"\").lower()\n subject = (e.get(\"subject\", \"\") or \"\")\n attachments = e.get(\"attachments\", []) or []\n if \"carlos.mendoza@prairiestar.com\" not in to_addr:\n continue\n # Check subject\n subj_ok = bool(re.search(r'IR-2026-04-01-Contamination(\\.pdf)?\\s+for\\s+review', subject, re.IGNORECASE))\n if not subj_ok:\n continue\n # Check attachment\n attach_ok = False\n for att in attachments:\n att_name = \"\"\n if isinstance(att, dict):\n att_name = att.get(\"filename\", \"\") or att.get(\"name\", \"\") or \"\"\n elif isinstance(att, str):\n att_name = att\n if \"IR-2026-04-01-Contamination\" in att_name:\n attach_ok = True\n break\n # Check body is blank/empty\n body = (e.get(\"body_text\", \"\") or \"\").strip()\n body_ok = len(body) == 0\n if subj_ok and attach_ok and body_ok:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found email to carlos.mendoza with subject '{subject}', blank body, and IR attachment\"}\n issues = []\n if not attach_ok:\n issues.append(f\"Missing IR-2026-04-01-Contamination.pdf attachment (found: {attachments})\")\n if not body_ok:\n issues.append(f\"Body not blank: '{body}'\")\n return {\"pass\": False, \"score\": 0.5, \"feedback\": f\"Found IR review email to carlos.mendoza but: {'; '.join(issues)}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email to carlos.mendoza@prairiestar.com with subject matching 'IR-2026-04-01-Contamination for review' found\"}",
|
||||
"criterion_type": "expected_output"
|
||||
},
|
||||
{
|
||||
"id": "a4be86d2-af0c-4e30-881b-e6f259724fa8",
|
||||
"sort_order": 9,
|
||||
"rubric_text": "In `mailbox.json`, there must be no reply to or forward of the email with subject 'Upcoming Server Update' from hannah.brooks@prairiestar.com",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n # Find the original server update email's message_id\n original_msg_id = None\n for e in emails:\n subject = e.get(\"subject\", \"\") or \"\"\n from_addr = e.get(\"from_addr\", \"\") or \"\"\n if \"upcoming server update\" in subject.lower() and \"hannah.brooks@prairiestar.com\" in from_addr.lower():\n original_msg_id = e.get(\"message_id\", \"\")\n break\n # Check no sent email replies to or forwards this email\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n subject = (e.get(\"subject\", \"\") or \"\").lower()\n in_reply_to = (e.get(\"in_reply_to\", \"\") or \"\").lower()\n # Check if it's a reply by in_reply_to field matching original message_id\n if original_msg_id and in_reply_to and original_msg_id.lower() in in_reply_to:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found a reply/forward to the 'Upcoming Server Update' email (via in_reply_to): '{e.get('subject', '')}'\"}\n # Check if subject looks like a reply or forward referencing server update\n if \"server update\" in subject:\n if \"re:\" in subject or \"fw:\" in subject or \"fwd:\" in subject:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found what appears to be a reply/forward to 'Upcoming Server Update' email: '{e.get('subject', '')}'\"}\n # Also catch cases where in_reply_to is set even without re:/fw: prefix\n if e.get(\"in_reply_to\"):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found what appears to be a reply to 'Upcoming Server Update' email (has in_reply_to set): '{e.get('subject', '')}'\"}\n # Also check if subject contains 'upcoming server update' with forward prefix\n if \"upcoming server update\" in subject:\n if \"fw:\" in subject or \"fwd:\" in subject or \"re:\" in subject:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found a reply/forward referencing 'Upcoming Server Update': '{e.get('subject', '')}'\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No reply to or forward of 'Upcoming Server Update' email from hannah.brooks@prairiestar.com found in Sent folder\"}\n",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "500679be-78dc-4566-9cda-7b53113a5eb2",
|
||||
"sort_order": 10,
|
||||
"rubric_text": "There must be exactly one file in the workspace with any variation of the name `inventory_master.xlsx`",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n workspace = Path(workspace_path)\n # Find all files matching inventory_master pattern\n matches = list(workspace.glob(\"*inventory*master*\")) + list(workspace.glob(\"*inventory_master*\"))\n # Deduplicate\n matches = list(set(matches))\n # Filter to files only\n matches = [f for f in matches if f.is_file()]\n if len(matches) == 1:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Exactly one inventory_master file found: '{matches[0].name}'\"}\n elif len(matches) == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No inventory_master file found in workspace\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Multiple inventory_master files found: {[f.name for f in matches]}\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "32091992-2b49-4d22-a54a-885049f48f16",
|
||||
"sort_order": 11,
|
||||
"rubric_text": "In `mailbox.json`, there must be no sent email to kevin.white@prairiestar.com",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for e in sent_emails:\n to_addr = (e.get(\"to_addr\", \"\") or \"\").lower()\n cc_addr = (e.get(\"cc_addr\", \"\") or \"\").lower()\n if \"kevin.white@prairiestar.com\" in to_addr or \"kevin.white@prairiestar.com\" in cc_addr:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found sent email to/cc kevin.white@prairiestar.com: subject='{e.get('subject', '')}'\"}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No sent email to kevin.white@prairiestar.com found\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "c023e4a4-31a4-4c1b-8c0c-079527ad3047",
|
||||
"sort_order": 12,
|
||||
"rubric_text": "In `temperature_log.xlsx`, cell D2 must be '-28 C'",
|
||||
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n files = list(Path(workspace_path).glob(\"*temperature_log*\"))\n if not files:\n files = list(Path(workspace_path).glob(\"*temp*log*\")) + list(Path(workspace_path).glob(\"*log*temp*\"))\n if not files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No temperature_log file found in workspace\"}\n wb = openpyxl.load_workbook(files[0])\n ws = wb.active\n d2 = ws[\"D2\"].value\n d2_str = str(d2).strip() if d2 is not None else \"\"\n # Accept -28 C, -28C, -28°C etc.\n import re\n normalized = re.sub(r'\\s+', ' ', d2_str).strip()\n if re.match(r'^-28\\s*[°]?\\s*[Cc]$', normalized):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"D2 = '{d2_str}' which matches -28 C\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"D2 expected '-28 C' but got '{d2_str}'\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "22121fad-e961-4671-886b-692a4cdc1688",
|
||||
"sort_order": 13,
|
||||
"rubric_text": "In `mailbox.json`, there must be a total of exactly 4 emails in the Sent folder",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path) as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n count = len(sent_emails)\n if count == 4:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 4 sent emails in mailbox.json\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 4 sent emails but found {count}. Subjects: {[e.get('subject','') for e in sent_emails]}\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
},
|
||||
{
|
||||
"id": "8bbdb09a-77eb-44dc-bcb6-34e0b6f0c2d2",
|
||||
"sort_order": 14,
|
||||
"rubric_text": "In `slack_data.json`, there must be a total of exactly 11 messages across all channels",
|
||||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n with open(slack_path) as f:\n data = json.load(f)\n messages = data.get(\"messages\", {})\n total = 0\n channel_counts = {}\n for channel_id, msgs in messages.items():\n count = len(msgs)\n total += count\n # Get channel name for feedback\n channels = data.get(\"channels\", {})\n ch_name = channels.get(channel_id, {}).get(\"name\", channel_id)\n channel_counts[ch_name] = count\n if total == 11:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found exactly 11 messages total across channels: {channel_counts}\"}\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 11 total messages but found {total}. Per channel: {channel_counts}\"}",
|
||||
"criterion_type": "incorrect_behavior"
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user