DerekSurge
80 lines
35 KiB
JSON
80 lines
35 KiB
JSON
[
|
|
{
|
|
"id": "df6c7406-ded9-4aee-834d-6fcbe184d505",
|
|
"sort_order": 0,
|
|
"rubric_text": "In `mailbox.json`, a sent email reply to elena.vasquez@ridgelinegear.com must exist that includes the text \"you are NOT currently eligible for FMLA leave for the following reason:\" and mentions hours 1,038 being less than 1,250, references form WH-381, includes the text \"if you would like, I can review your FAMLI eligibility for the same leave reason\", and CCs renata.okafor@ridgelinegear.com",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n \n # Find emails TO Elena (sent by the leaves inbox or any agent address)\n elena_sent = [e for e in emails \n if \"elena.vasquez@ridgelinegear.com\" in (e.get(\"to_addr\", \"\") or \"\").lower()]\n \n if not elena_sent:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email to elena.vasquez@ridgelinegear.com found\"}\n \n checks = []\n for email in elena_sent:\n body = (email.get(\"body_text\", \"\") or \"\")\n body_lower = body.lower()\n cc = (email.get(\"cc_addr\", \"\") or \"\").lower()\n \n # Check 1: FMLA ineligibility language - be loose about exact phrasing\n # The rubric says: \"you are NOT currently eligible for FMLA leave for the following reason:\"\n # Accept variations: with/without colon, slight wording differences\n has_not_eligible = False\n not_eligible_patterns = [\n \"not currently eligible for fmla\",\n \"not eligible for fmla\",\n \"ineligible for fmla\",\n \"do not meet the eligibility requirements for fmla\",\n \"do not meet the fmla eligibility\",\n \"does not meet the eligibility requirements for fmla\",\n \"not qualify for fmla\",\n \"not meet the eligibility criteria for fmla\",\n \"you are not currently eligible for fmla leave\",\n \"you are not eligible for fmla\",\n ]\n for pattern in not_eligible_patterns:\n if pattern in body_lower:\n has_not_eligible = True\n break\n \n # Check 2: hours 1,038 being less than 1,250\n # Accept various formats: 1,038 or 1038 and 1,250 or 1250\n has_hours = False\n has_1038 = bool(re.search(r'1[,.]?038', body))\n has_1250 = bool(re.search(r'1[,.]?250', body))\n if has_1038 and has_1250:\n has_hours = True\n \n # Check 3: references form WH-381\n has_wh381 = bool(re.search(r'wh[\\s-]?381', body_lower))\n \n # Check 4: FAMLI eligibility review offer\n # The rubric says: \"if you would like, I can review your FAMLI eligibility for the same leave reason\"\n # Be loose about exact phrasing\n has_famli_offer = False\n famli_patterns = [\n \"review your famli eligibility\",\n \"famli eligibility for the same leave reason\",\n \"review your famli\",\n \"check your famli eligibility\",\n \"assess your famli eligibility\",\n \"evaluate your famli eligibility\",\n \"look into your famli eligibility\",\n \"consider your famli eligibility\",\n \"explore your famli eligibility\",\n \"famli eligibility\",\n ]\n for pattern in famli_patterns:\n if pattern in body_lower:\n has_famli_offer = True\n break\n \n # Check 5: CCs renata.okafor@ridgelinegear.com\n has_cc = \"renata.okafor@ridgelinegear.com\" in cc\n \n score_count = sum([has_not_eligible, has_hours, has_wh381, has_famli_offer, has_cc])\n \n checks.append({\n \"email_id\": email.get(\"email_id\"),\n \"has_not_eligible\": has_not_eligible,\n \"has_hours\": has_hours,\n \"has_wh381\": has_wh381,\n \"has_famli_offer\": has_famli_offer,\n \"has_cc\": has_cc,\n \"score_count\": score_count\n })\n \n # Sort by most checks passed to evaluate the best candidate\n checks.sort(key=lambda c: c[\"score_count\"], reverse=True)\n best = checks[0]\n \n if best[\"score_count\"] == 5:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found qualifying email (id={best['email_id']}) to Elena with all required content: FMLA ineligibility language, hours comparison (1,038 < 1,250), WH-381 reference, FAMLI eligibility review offer, and CC to renata.okafor@ridgelinegear.com\"}\n \n missing = []\n if not best[\"has_not_eligible\"]: missing.append(\"FMLA ineligibility language (e.g. 'not currently eligible for FMLA')\")\n if not best[\"has_hours\"]: missing.append(\"hours comparison (1,038 < 1,250)\")\n if not best[\"has_wh381\"]: missing.append(\"WH-381 form reference\")\n if not best[\"has_famli_offer\"]: missing.append(\"FAMLI eligibility review offer\")\n if not best[\"has_cc\"]: missing.append(\"CC to renata.okafor@ridgelinegear.com\")\n \n return {\"pass\": False, \"score\": round(best[\"score_count\"] / 5, 2), \"feedback\": f\"Best email (id={best['email_id']}) to Elena is missing: {', '.join(missing)}\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "0cb1e3ff-3174-4aaa-aa76-6b92ca8b8bac",
|
|
"sort_order": 1,
|
|
"rubric_text": "In `mailbox.json`, the sent email to elena.vasquez@ridgelinegear.com must include a Case ID in the format LC-2026-NNNN where NNNN is a number greater than 0320",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n \n with open(mailbox_path) as f:\n data = json.load(f)\n \n emails = data.get(\"emails\", [])\n elena_sent = [e for e in emails \n if \"leaves@ridgelinegear.com\" in (e.get(\"from_addr\", \"\") or \"\").lower()\n and \"elena.vasquez@ridgelinegear.com\" in (e.get(\"to_addr\", \"\") or \"\").lower()]\n \n if not elena_sent:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No email from leaves@ridgelinegear.com to elena.vasquez@ridgelinegear.com found\"}\n \n for email in elena_sent:\n body = email.get(\"body_text\", \"\") or \"\"\n subject = email.get(\"subject\", \"\") or \"\"\n combined = body + \" \" + subject\n \n matches = re.findall(r'LC-2026-(\\d+)', combined, re.IGNORECASE)\n for m in matches:\n if int(m) > 320:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found Case ID LC-2026-{m} in email to Elena (NNNN={m} > 0320)\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Case ID in format LC-2026-NNNN (with NNNN > 0320) found in email to Elena\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "81f65033-2847-4b15-81c6-f8bc947fda30",
|
|
"sort_order": 2,
|
|
"rubric_text": "In mailbox.json, email 35 from elena.vasquez@ridgelinegear.com must be in the inbox.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n\n with open(mailbox_path) as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n\n # Find email with id=35 (accept both int and string)\n target_email = None\n for email in emails:\n email_id = email.get(\"email_id\")\n if str(email_id) == \"35\" or email_id == 35:\n target_email = email\n break\n\n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email with id 35 not found in mailbox.json. Total emails: {len(emails)}.\"}\n\n folder = (target_email.get(\"folder\", \"\") or \"\").strip()\n if folder.lower() == \"inbox\":\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email 35 is correctly in the INBOX folder (folder='{folder}'). Subject: {target_email.get('subject', 'N/A')}.\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email 35 is in folder '{folder}', expected 'INBOX'. Subject: {target_email.get('subject', 'N/A')}.\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "d38460f9-0c67-4cb4-93c2-b2ef1fc5618d",
|
|
"sort_order": 3,
|
|
"rubric_text": "In `leave_tracker.xlsx`, row 12 must contain the following values: A12=LC-2026-NNNN (NNNN > 0320), B12=R-2317, C12=Elena Vasquez, D12=FMLA, E12=04/20/2026, F12=05/15/2026, H12=Continuous, N12=P2, O12=Denied",
|
|
"verifier_code": "from pathlib import Path\nimport re\nfrom datetime import datetime, date\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n tracker_files = [f for f in xlsx_files if \"leave_tracker\" in f.name.lower()]\n\n if not tracker_files:\n tracker_files = [f for f in xlsx_files if \"leave\" in f.name.lower()]\n\n if not tracker_files:\n tracker_files = xlsx_files\n\n if not tracker_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace\"}\n\n tracker_path = tracker_files[0]\n\n try:\n wb = openpyxl.load_workbook(tracker_path)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening xlsx: {e}\"}\n\n def get_cell_value(ws, cell_ref):\n return ws[cell_ref].value\n\n issues = []\n total_checks = 9\n\n # A12: LC-2026-NNNN with NNNN > 0320\n a12 = str(get_cell_value(ws, \"A12\") or \"\")\n m = re.search(r'LC-2026-(\\d+)', a12, re.IGNORECASE)\n if not m or int(m.group(1)) <= 320:\n issues.append(f\"A12='{a12}' should be LC-2026-NNNN with NNNN > 0320\")\n\n # B12: R-2317\n b12 = str(get_cell_value(ws, \"B12\") or \"\").strip()\n if \"r-2317\" not in b12.lower() and \"r2317\" not in b12.lower():\n issues.append(f\"B12='{b12}' should be R-2317\")\n\n # C12: Elena Vasquez\n c12 = str(get_cell_value(ws, \"C12\") or \"\").strip().lower()\n if \"elena\" not in c12 or \"vasquez\" not in c12:\n issues.append(f\"C12='{c12}' should be Elena Vasquez\")\n\n # D12: FMLA\n d12 = str(get_cell_value(ws, \"D12\") or \"\").strip().lower()\n if \"fmla\" not in d12:\n issues.append(f\"D12='{d12}' should be FMLA\")\n\n # E12: April 20, 2026\n e12 = get_cell_value(ws, \"E12\")\n e12_ok = False\n if isinstance(e12, (datetime, date)):\n if hasattr(e12, 'year'):\n e12_ok = (e12.month == 4 and e12.day == 20 and e12.year == 2026)\n elif e12 is not None:\n e12_str = str(e12).lower()\n e12_ok = (\"04/20/2026\" in e12_str or \"4/20/2026\" in e12_str)\n if not e12_ok:\n issues.append(f\"E12='{e12}' should be 04/20/2026\")\n\n # F12: May 15, 2026\n f12 = get_cell_value(ws, \"F12\")\n f12_ok = False\n if isinstance(f12, (datetime, date)):\n if hasattr(f12, 'year'):\n f12_ok = (f12.month == 5 and f12.day == 15 and f12.year == 2026)\n elif f12 is not None:\n f12_str = str(f12).lower()\n f12_ok = (\"05/15/2026\" in f12_str or \"5/15/2026\" in f12_str)\n if not f12_ok:\n issues.append(f\"F12='{f12}' should be 05/15/2026\")\n\n # H12: Continuous\n h12 = str(get_cell_value(ws, \"H12\") or \"\").strip().lower()\n if \"continuous\" not in h12:\n issues.append(f\"H12='{h12}' should be Continuous\")\n\n # N12: P2\n n12 = str(get_cell_value(ws, \"N12\") or \"\").strip().lower()\n if \"p2\" not in n12:\n issues.append(f\"N12='{n12}' should be P2\")\n\n # O12: Denied\n o12 = str(get_cell_value(ws, \"O12\") or \"\").strip().lower()\n if \"denied\" not in o12:\n issues.append(f\"O12='{o12}' should be Denied\")\n\n passed_checks = total_checks - len(issues)\n score = max(0.0, passed_checks / total_checks)\n\n if issues:\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 12 issues in {tracker_path.name} ({passed_checks}/{total_checks} checks passed): \" + \"; \".join(issues)\n }\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Row 12 in {tracker_path.name} contains all required values for Elena Vasquez's denied FMLA claim (A12 case ID OK, B12=R-2317, C12=Elena Vasquez, D12=FMLA, E12=04/20/2026, F12=05/15/2026, H12=Continuous, N12=P2, O12=Denied)\"\n }",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "bd125ee1-0a13-4442-922f-4ca62a06c996",
|
|
"sort_order": 4,
|
|
"rubric_text": "In the Leave Tracker worksheet of leave_tracker.xlsx, row 12 cells G12, J12, K12, L12, M12 must be empty or contain a non-applicable indicator (e.g., \"N/A\", \"Not Required\", \"-\"), and I12 must be empty, 0, or a non-applicable indicator.",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n tracker_files = [f for f in xlsx_files if \"leave_tracker\" in f.name.lower()]\n\n if not tracker_files:\n tracker_files = [f for f in xlsx_files if \"leave\" in f.name.lower()]\n if not tracker_files:\n tracker_files = xlsx_files\n if not tracker_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found\"}\n\n tracker_path = tracker_files[0]\n try:\n wb = openpyxl.load_workbook(tracker_path)\n ws = None\n for name in wb.sheetnames:\n if \"leave\" in name.lower() and \"tracker\" in name.lower():\n ws = wb[name]\n break\n if ws is None:\n for name in wb.sheetnames:\n if \"leave\" in name.lower():\n ws = wb[name]\n break\n if ws is None:\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening xlsx: {e}\"}\n\n def is_empty_or_na(val):\n if val is None:\n return True\n s = str(val).strip().lower()\n return s in (\"\", \"n/a\", \"na\", \"none\", \"-\", \"--\", \"0\", \"not required\", \"not applicable\", \"nr\", \"n\", \"no\", \"not notified\")\n\n def is_empty_zero_or_na(val):\n if val is None:\n return True\n s = str(val).strip().lower()\n if s in (\"\", \"n/a\", \"na\", \"none\", \"-\", \"--\", \"not required\", \"not applicable\", \"nr\", \"n\", \"no\", \"not notified\"):\n return True\n try:\n return float(s) == 0\n except (ValueError, TypeError):\n return False\n\n issues = []\n found_values = {}\n\n for col in [\"G12\", \"J12\", \"K12\", \"L12\", \"M12\"]:\n val = ws[col].value\n found_values[col] = val\n if not is_empty_or_na(val):\n issues.append(f\"{col}='{val}' should be empty, N/A, or 'not required'\")\n\n i12 = ws[\"I12\"].value\n found_values[\"I12\"] = i12\n if not is_empty_zero_or_na(i12):\n issues.append(f\"I12='{i12}' should be empty, 0, or N/A\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Row 12 cells not matching expected empty/NA state: \" + \"; \".join(issues) + f\". Found values: {found_values}. Sheet: '{ws.title}' in {tracker_path.name}\"}\n\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"G12, I12, J12, K12, L12, M12 are all empty/N/A/0/'not required' as required. Found values: {found_values}. Sheet: '{ws.title}' in {tracker_path.name}\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "02b7d23d-d9e2-43df-99bc-1e91f5a63ea2",
|
|
"sort_order": 5,
|
|
"rubric_text": "In `leave_tracker.xlsx`, cell G11 must equal 04/10/2026, O11 must equal 'Closed', and P11 must mention that the case was closed on 04/10 or 04/10/2026.",
|
|
"verifier_code": "from pathlib import Path\nfrom datetime import datetime, date\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n tracker_files = [f for f in xlsx_files if \"leave_tracker\" in f.name.lower()]\n if not tracker_files:\n tracker_files = [f for f in xlsx_files if \"leave\" in f.name.lower()]\n if not tracker_files:\n tracker_files = xlsx_files\n if not tracker_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace\"}\n\n tracker_path = tracker_files[0]\n try:\n wb = openpyxl.load_workbook(tracker_path)\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening xlsx: {e}\"}\n\n issues = []\n\n # G11: April 10, 2026 — accept datetime object or MM/DD/YYYY string\n g11 = ws[\"G11\"].value\n g11_ok = False\n if isinstance(g11, (datetime, date)):\n g11_ok = (g11.month == 4 and g11.day == 10 and g11.year == 2026)\n elif g11 is not None:\n g11_str = str(g11).strip()\n g11_ok = g11_str in (\"04/10/2026\", \"4/10/2026\")\n if not g11_ok:\n issues.append(f\"G11='{g11}' expected 04/10/2026\")\n\n # O11: Closed\n o11_raw = ws[\"O11\"].value\n o11 = str(o11_raw or \"\").strip().lower()\n if \"closed\" not in o11:\n issues.append(f\"O11='{o11_raw}' expected 'Closed'\")\n\n # P11: 'closed' must appear within 20 characters of 04/10/2026 or 4/10/2026\n close_word = r\"clos(?:ed|ing|ure|e)\"\n date_pat = r\"(?:04/10/2026|4/10/2026)\"\n pat1 = close_word + r\"[\\s\\w,\\.:\\-]{0,20}?\" + date_pat\n pat2 = date_pat + r\".{0,15}\" + close_word\n\n p11_raw = ws[\"P11\"].value\n p11_lower = str(p11_raw or \"\").lower()\n p11_ok = bool(re.search(pat1, p11_lower) or re.search(pat2, p11_lower))\n\n if not p11_ok:\n issues.append(\n f\"P11='{str(p11_raw)[:200]}' — 'closed' must appear adjacent to 04/10/2026 \"\n f\"(e.g. 'Case closed 04/10/2026')\"\n )\n\n if issues:\n score = max(0.0, 1.0 - len(issues) * 0.34)\n return {\"pass\": False, \"score\": round(score, 2),\n \"feedback\": f\"Row 11 update issues in '{tracker_path.name}': \" + \"; \".join(issues)}\n\n return {\"pass\": True, \"score\": 1.0,\n \"feedback\": (f\"All checks passed in '{tracker_path.name}': G11={g11} (04/10/2026), \"\n f\"O11=Closed, P11 mentions case closed on 04/10/2026\")}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "18444be2-67ff-4342-835d-666cea89624d",
|
|
"sort_order": 7,
|
|
"rubric_text": "In `slack_data.json`, the #leave-intake channel must have a new message containing 'FMLA DENIED' and 'Elena Vasquez' and mentioning insufficient hours (referencing 1,038 or similar) with a Case ID LC-2026-NNNN where NNNN > 0320",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n\n with open(slack_path) as f:\n data = json.load(f)\n\n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n\n # Find #leave-intake channel\n leave_intake_id = None\n for ch_id, ch_info in channels.items():\n if \"leave-intake\" in ch_info.get(\"name\", \"\").lower():\n leave_intake_id = ch_id\n break\n\n if leave_intake_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"#leave-intake channel not found in slack_data.json\"}\n\n channel_msgs = messages.get(leave_intake_id, [])\n\n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\").lower()\n\n has_denied = \"fmla denied\" in text or (\"fmla\" in text and \"denied\" in text)\n has_elena = \"elena vasquez\" in text or (\"elena\" in text and \"vasquez\" in text)\n has_hours = (\"1,038\" in text or \"1038\" in text or\n \"insufficient hours\" in text or \"hours worked\" in text or\n \"1,250\" in text or\n (\"hours\" in text and (\"below\" in text or \"less\" in text or\n \"short\" in text or \"insufficient\" in text)))\n case_id_match = re.search(r'lc-2026-(\\d+)', text, re.IGNORECASE)\n has_case_id = case_id_match and int(case_id_match.group(1)) > 320\n\n if has_denied and has_elena and has_hours and has_case_id:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found qualifying FMLA DENIED message in #leave-intake for Elena Vasquez with Case ID LC-2026-{case_id_match.group(1)}\"}\n\n if channel_msgs:\n sample = channel_msgs[-1].get(\"text\", \"\")[:200]\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No qualifying FMLA DENIED message found in #leave-intake. Last message sample: '{sample}'\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in #leave-intake channel\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "a220c670-6076-4416-aac2-ff187e3066fe",
|
|
"sort_order": 8,
|
|
"rubric_text": "In `slack_data.json`, the #payroll-coordination channel must have a new message starting with the exact text: 'LEAVE PAYROLL NOTIFICATION:' followed by 'LC-2026-0320, Marcus DeLuca' and must include the date 04/10/2026, 'Continuous' and an event type indicating return from leave or return from jury duty.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n \n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n \n with open(slack_path) as f:\n data = json.load(f)\n \n channels = data.get(\"channels\", {})\n messages = data.get(\"messages\", {})\n \n # Find #payroll-coordination channel\n payroll_id = None\n for ch_id, ch_info in channels.items():\n if \"payroll\" in ch_info.get(\"name\", \"\").lower():\n payroll_id = ch_id\n break\n \n if payroll_id is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"#payroll-coordination channel not found\"}\n \n channel_msgs = messages.get(payroll_id, [])\n \n # Return event type can be phrased in various ways\n return_phrases = [\n \"return from leave\",\n \"return from jury duty\",\n \"return to work\",\n \"scheduled return\",\n \"confirmed return\",\n \"returned from leave\",\n \"returned from jury duty\",\n \"jury duty return\",\n \"leave return\",\n \"end of leave\",\n \"end of jury duty\"\n ]\n \n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\")\n text_lower = text.lower()\n \n # Normalize date formats for comparison\n text_lower_normalized = text_lower.replace(\"4/10/2026\", \"04/10/2026\")\n \n # Check starts with LEAVE PAYROLL NOTIFICATION:\n if not text_lower_normalized.strip().startswith(\"leave payroll notification:\"):\n continue\n \n # Check for case ID\n if \"lc-2026-0320\" not in text_lower_normalized:\n continue\n \n # Check for employee name\n if \"marcus deluca\" not in text_lower_normalized:\n continue\n \n # Check for date\n if \"04/10/2026\" not in text_lower_normalized:\n continue\n \n # Check for Continuous\n if \"continuous\" not in text_lower_normalized:\n continue\n \n # Check for return event type\n has_return = any(phrase in text_lower_normalized for phrase in return_phrases)\n if not has_return:\n continue\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found qualifying payroll notification message in #payroll-coordination for Marcus DeLuca: '{text[:300]}'\"}\n \n # Provide detailed failure feedback\n all_msgs_text = []\n for msg in channel_msgs:\n t = (msg.get(\"text\", \"\") or \"\")\n if t:\n all_msgs_text.append(t[:300])\n \n if channel_msgs:\n # Check the most recent messages to see what's missing\n diagnostics = []\n for msg in channel_msgs:\n text = (msg.get(\"text\", \"\") or \"\").lower()\n text = text.replace(\"4/10/2026\", \"04/10/2026\")\n if \"leave payroll notification\" in text or \"lc-2026-0320\" in text or \"marcus deluca\" in text:\n missing = []\n if not text.strip().startswith(\"leave payroll notification:\"):\n missing.append(\"starts with 'LEAVE PAYROLL NOTIFICATION:'\")\n if \"lc-2026-0320\" not in text:\n missing.append(\"LC-2026-0320\")\n if \"marcus deluca\" not in text:\n missing.append(\"Marcus DeLuca\")\n if \"04/10/2026\" not in text:\n missing.append(\"04/10/2026\")\n if \"continuous\" not in text:\n missing.append(\"Continuous\")\n has_return = any(phrase in text for phrase in return_phrases)\n if not has_return:\n missing.append(\"return from leave/jury duty event type\")\n diagnostics.append(f\"Partial match found but missing: {missing}. Text: '{msg.get('text', '')[:300]}'\")\n \n if diagnostics:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No fully qualifying payroll notification found. {'; '.join(diagnostics)}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No qualifying payroll notification found in #payroll-coordination. Messages found: {all_msgs_text[:3]}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages in #payroll-coordination channel\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775867812527",
|
|
"sort_order": 9,
|
|
"rubric_text": "In `mailbox.json`, email 35 must be marked as read (is_read=True)",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n feedback_parts = []\n score = 0.0\n\n if external_services_path is None:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"No external_services_path provided; cannot verify mailbox.json.\"\n }\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n\n if not mailbox_path.exists():\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"mailbox.json not found at expected path.\"\n }\n\n try:\n with open(mailbox_path, 'r') as f:\n mailbox_data = json.load(f)\n except Exception as e:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Error reading mailbox.json: {e}\"\n }\n\n emails = mailbox_data.get(\"emails\", [])\n\n # Find email with id 35 (accept both int and string)\n target_email = None\n for email in emails:\n email_id = email.get(\"email_id\")\n if str(email_id) == \"35\" or email_id == 35:\n target_email = email\n break\n\n if target_email is None:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Email with id 35 not found in mailbox.json. Total emails found: {len(emails)}.\"\n }\n\n is_read = target_email.get(\"is_read\")\n\n # Accept truthy values: True, \"true\", \"True\", 1, \"1\", etc.\n if is_read is True or (isinstance(is_read, str) and is_read.lower() == \"true\") or is_read == 1:\n score = 1.0\n feedback_parts.append(f\"Email 35 is correctly marked as read (is_read={is_read}). Subject: {target_email.get('subject', 'N/A')}.\")\n return {\n \"pass\": True,\n \"score\": score,\n \"feedback\": \" | \".join(feedback_parts)\n }\n else:\n feedback_parts.append(f\"Email 35 is NOT marked as read. Expected is_read=True but found is_read={is_read}. Subject: {target_email.get('subject', 'N/A')}.\")\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \" | \".join(feedback_parts)\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775868072725",
|
|
"sort_order": 10,
|
|
"rubric_text": "In `mailbox.json`, email 16 must be in the 'Closed Cases' folder.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external services path for mailbox.json\n mailbox_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n # Also check workspace path as fallback\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find mailbox.json in external_services_path or workspace_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n if not emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in mailbox.json.\"}\n \n # Find email with email_id 16 (check both string and int representations)\n target_email = None\n for email in emails:\n eid = email.get(\"email_id\", \"\")\n if str(eid).strip() == \"16\":\n target_email = email\n break\n \n if target_email is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Email with email_id 16 was not found in mailbox.json.\"}\n \n folder = target_email.get(\"folder\", \"\")\n # Be generous: case-insensitive comparison, strip whitespace\n folder_normalized = folder.strip().lower()\n expected_normalized = \"closed cases\"\n \n if folder_normalized == expected_normalized:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Email 16 is in the '{folder}' folder as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Email 16 is in folder '{folder}', but expected 'Closed Cases'.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1775869133744",
|
|
"sort_order": 11,
|
|
"rubric_text": "In `leave_tracker.xlsx`, row 12 in the \"Leave Tracker\" sheet must not be empty.",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n \n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\"))\n tracker_files = [f for f in xlsx_files if \"leave_tracker\" in f.name.lower()]\n \n if not tracker_files:\n tracker_files = [f for f in xlsx_files if \"leave\" in f.name.lower()]\n \n if not tracker_files:\n tracker_files = xlsx_files\n \n if not tracker_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No xlsx file found in workspace\"}\n \n tracker_path = tracker_files[0]\n try:\n wb = openpyxl.load_workbook(str(tracker_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open '{tracker_path.name}': {e}\"}\n \n # Try named sheet first, fall back to active\n target_sheet = None\n for name in wb.sheetnames:\n if name.strip().lower() == \"leave tracker\":\n target_sheet = wb[name]\n break\n \n if target_sheet is None:\n target_sheet = wb.active\n \n row_12_values = []\n for cell in target_sheet[12]:\n if cell.value is not None and str(cell.value).strip() != \"\":\n row_12_values.append((cell.column_letter, cell.value))\n \n if len(row_12_values) == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 12 in '{tracker_path.name}' is completely empty.\"}\n \n values_summary = \", \".join([f\"{col}: {val}\" for col, val in row_12_values[:10]])\n if len(row_12_values) > 10:\n values_summary += f\" ... and {len(row_12_values) - 10} more\"\n \n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 12 is not empty. Found {len(row_12_values)} non-empty cell(s): {values_summary}\"}",
|
|
"criterion_type": "expected_output"
|
|
}
|
|
]
|