DerekSurge
101 lines
88 KiB
JSON
101 lines
88 KiB
JSON
[
|
|
{
|
|
"id": "c74f7f2a-95ff-458c-9a03-8943860fcaef",
|
|
"sort_order": 0,
|
|
"rubric_text": "In mailbox.json (Sent folder), a sent email to sofia.ramirez@sunshineandsetauto.com must exist with the subject line 'Deal Gross Variance — Deal #10057 / Flores' and with body text \ncontaining the exact text 'Deal #10057', 'Flores', 'Stock #N16028', 'VIN ending '342FK', 'Approved gross: $7,500', 'Posted gross: $6,700', 'Variance: $800', and 'Source of difference', with Source of difference referencing accessory charge or desk recap error.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent emails found in mailbox.json.\"}\n \n # Primary target recipient per rubric\n target_recipients = [\n \"sofia.ramirez@sunshineandsetauto.com\",\n \"sofia.ramirez\",\n \"sofia\"\n ]\n \n best_match_email = None\n best_missing = None\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or email.get(\"cc\") or \"\").lower()\n bcc_addr = (email.get(\"bcc_addr\") or email.get(\"bcc\") or \"\").lower()\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n combined_addr = to_addr + \" \" + cc_addr + \" \" + bcc_addr\n combined_text = body + \" \" + subject\n \n # Check if sent to sofia.ramirez (primary requirement per rubric)\n recipient_match = any(r.lower() in combined_addr for r in target_recipients)\n if not recipient_match:\n continue\n \n missing = []\n \n # Check subject line contains deal gross variance and deal #10057 and Flores\n # Be lenient: accept various dash types (-, \\u2014, \\u2013) or no dash\n subject_has_deal_gross_variance = re.search(r'deal\\s*gross\\s*variance', subject)\n subject_has_10057 = re.search(r'10057', subject)\n subject_has_flores = 'flores' in subject\n if not subject_has_deal_gross_variance:\n missing.append(\"Subject must contain 'Deal Gross Variance'\")\n if not subject_has_10057:\n missing.append(\"Subject must contain 'Deal #10057'\")\n if not subject_has_flores:\n missing.append(\"Subject must contain 'Flores'\")\n \n # Deal #10057 in body - accept various formats\n if not re.search(r'10057', combined_text):\n missing.append(\"Body: Deal #10057\")\n \n # Flores in body\n if 'flores' not in combined_text:\n missing.append(\"Body: Flores\")\n \n # Stock #N16028 - accept N16028, N-16028, N 16028, etc.\n if not re.search(r'n[\\s\\-#]*16028', combined_text):\n missing.append(\"Body: Stock #N16028\")\n \n # VIN ending 342FK\n if not re.search(r'342fk', combined_text):\n missing.append(\"Body: VIN ending 342FK\")\n \n # Approved gross $7,500 - accept 7500, 7,500, 7500.00 etc.\n if not re.search(r'7[,.]?500', combined_text):\n missing.append(\"Body: Approved gross: $7,500\")\n \n # Posted gross $6,700 - accept 6700, 6,700, 6700.00 etc.\n if not re.search(r'6[,.]?700', combined_text):\n missing.append(\"Body: Posted gross: $6,700\")\n \n # Variance $800\n if not re.search(r'800', combined_text):\n missing.append(\"Body: Variance: $800\")\n \n # Source of difference - accessory charge or desk recap error\n accessory_keywords = [\"accessory\", \"accessories\", \"desk recap\", \"recap error\", \"recap\", \"add-on\", \"addon\", \"aftermarket\", \"source of difference\", \"discrepancy\", \"difference\", \"source\"]\n has_source = any(kw in combined_text for kw in accessory_keywords)\n if not has_source:\n missing.append(\"Body: Source of difference (accessory charge / desk recap error)\")\n \n if not missing:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found email sent to sofia.ramirez@sunshineandsetauto.com with subject containing 'Deal Gross Variance \\u2014 Deal #10057 / Flores' and all required body fields: Deal #10057, Flores, Stock #N16028, VIN ending 342FK, Approved gross: $7,500, Posted gross: $6,700, Variance: $800, and Source of difference referencing accessory charge or desk recap error.\"}\n \n # Track best partial match\n if best_missing is None or len(missing) < len(best_missing):\n best_missing = missing\n best_match_email = email\n \n if best_missing is not None:\n total_checks = 11 # 3 subject checks + 8 body checks\n score = max(0.1, 1.0 - (len(best_missing) / total_checks))\n return {\"pass\": False, \"score\": round(score, 2), \"feedback\": f\"Found email to sofia.ramirez but missing required fields: {', '.join(best_missing)}\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to sofia.ramirez@sunshineandsetauto.com with Deal Gross Variance details for Deal #10057 / Flores.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "25622b64-4ad0-41ae-9ed5-54e0a58e4fb2",
|
|
"sort_order": 1,
|
|
"rubric_text": "In the deal_log spreadsheet, row for Deal #10057 must contain text in the 'Notes' column (column AH) that states that the desk recap was in error and the deal_log total gross of $6,700 is correct (referencing Marcus Hale's resolution).",
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available.\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\")) + list(Path(workspace_path).glob(\"*.xlsm\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Excel files found in workspace.\"}\n\n deal_log_files = [f for f in xlsx_files if \"deal\" in f.name.lower() and \"log\" in f.name.lower()]\n if not deal_log_files:\n deal_log_files = xlsx_files\n\n # Keywords indicating the desk recap was in error (REQUIRED along with $6,700)\n keywords_error = [\"error\", \"incorrect\", \"in error\", \"was in error\", \"discrepancy\", \"wrong\", \"corrected\", \"correction\", \"mismatch\", \"adjusted\", \"override\", \"overridden\", \"resolved\", \"updated\", \"issue\", \"differs\", \"difference\"]\n # Keywords indicating $6,700 / deal_log is correct (REQUIRED)\n keywords_correct_amount = [\"6,700\", \"6700\", \"$6,700\", \"$6700\"]\n # Keywords referencing that the amount/deal_log is correct (supporting signal)\n keywords_correct_source = [\"correct\", \"deal_log\", \"deal log\", \"confirmed\", \"verified\", \"accurate\", \"actual\", \"true\", \"valid\", \"approved\", \"stands\", \"retained\"]\n # Keywords referencing Marcus Hale (supporting signal)\n keywords_marcus = [\"marcus\", \"hale\"]\n # Keywords referencing desk recap (supporting signal)\n keywords_recap = [\"recap\", \"desk\"]\n\n def _cell_str(v):\n return str(v).lower() if v is not None else \"\"\n\n def _find_col_index(header_row, candidates_substrings):\n \"\"\"Return 0-based index of first header cell whose lowercased text contains any candidate substring.\"\"\"\n for idx, cell in enumerate(header_row):\n val = _cell_str(cell.value).strip()\n if not val:\n continue\n for cand in candidates_substrings:\n if cand in val:\n return idx\n return None\n\n def _row_matches_deal(row, deal_col_idx, deal_number):\n if deal_col_idx is None or deal_col_idx >= len(row):\n return False\n raw = row[deal_col_idx].value\n if raw is None:\n return False\n try:\n return int(float(raw)) == deal_number\n except (TypeError, ValueError):\n return str(raw).strip() == str(deal_number)\n\n all_found_notes = []\n\n for fpath in deal_log_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n except Exception:\n continue\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows())\n if not rows:\n continue\n\n header_row = rows[0]\n\n # Locate Notes column (prefer exact \"notes\" header)\n notes_col_idx = None\n for idx, cell in enumerate(header_row):\n val = _cell_str(cell.value).strip()\n if val == \"notes\" or val == \"note\" or val == \"comments\":\n notes_col_idx = idx\n break\n # Fallback: any header containing \"note\"\n if notes_col_idx is None:\n notes_col_idx = _find_col_index(header_row, [\"note\"])\n\n # Locate Deal # column\n deal_col_idx = None\n for idx, cell in enumerate(header_row):\n val = _cell_str(cell.value).strip()\n if val in (\"deal #\", \"deal#\", \"deal number\", \"deal no\", \"deal no.\", \"deal\"):\n deal_col_idx = idx\n break\n\n # Fallback column AH (index 33) if Notes header not found\n ah_col_idx = 33\n\n # If we can't find a deal column on this sheet, skip\n if deal_col_idx is None:\n continue\n\n for row in rows[1:]:\n if not _row_matches_deal(row, deal_col_idx, 10057):\n continue\n\n # Extract the Notes cell text only (scoped to the column)\n notes_text = \"\"\n if notes_col_idx is not None and notes_col_idx < len(row):\n notes_text = _cell_str(row[notes_col_idx].value)\n\n # Fallback: column AH if Notes column wasn't found or is empty\n ah_text = \"\"\n if ah_col_idx < len(row):\n ah_text = _cell_str(row[ah_col_idx].value)\n\n # The text we evaluate must come from the Notes column (or AH fallback),\n # NOT from the entire row — otherwise keyword matches can come from\n # unrelated cells like F&I Manager (\"Marcus Hale\") or amount columns.\n check_text = (notes_text + \" \" + ah_text).strip()\n\n # Reject placeholder-only content\n cleaned = check_text.replace(\"none\", \"\").replace(\"nan\", \"\").strip()\n if len(cleaned) <= 3:\n continue\n\n all_found_notes.append(check_text)\n\n has_error = any(kw in check_text for kw in keywords_error)\n has_recap = any(kw in check_text for kw in keywords_recap)\n has_correct_amount = any(kw in check_text for kw in keywords_correct_amount)\n has_correct_source = any(kw in check_text for kw in keywords_correct_source)\n has_marcus = any(kw in check_text for kw in keywords_marcus)\n\n # HARD REQUIREMENT: $6,700 AND an error-indicator keyword must both appear\n # in the Notes column text. No pass without both.\n if not (has_correct_amount and has_error):\n continue\n\n # Full pass: 6,700 + error + Marcus reference\n if has_marcus:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found Deal #10057 row in '{fpath.name}' sheet '{sheet_name}' with Notes indicating desk recap was in error, $6,700 is correct, and referencing Marcus Hale. Notes: {check_text[:300]}\"}\n\n # Good pass: 6,700 + error + desk recap reference\n if has_recap:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found Deal #10057 row in '{fpath.name}' sheet '{sheet_name}' with Notes indicating desk recap error and $6,700 correct. Notes: {check_text[:300]}\"}\n\n # Acceptable pass: 6,700 + error + correctness signal\n if has_correct_source:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found Deal #10057 row in '{fpath.name}' sheet '{sheet_name}' with Notes indicating error and $6,700 being correct. Notes: {check_text[:300]}\"}\n\n # Partial: 6,700 + error but no Marcus/recap/correctness context\n return {\"pass\": True, \"score\": 0.8, \"feedback\": f\"Found Deal #10057 row in '{fpath.name}' sheet '{sheet_name}' with $6,700 and error-indicator keywords but limited context. Notes: {check_text[:300]}\"}\n\n if all_found_notes:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found Deal #10057 row(s) but Notes column did not contain both $6,700 AND an error-indicator keyword as required. Found Notes text: {'; '.join(n[:200] for n in all_found_notes)}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No row found for Deal #10057 in any deal_log spreadsheet with Notes about desk recap error and $6,700 being correct. Files checked: {[f.name for f in deal_log_files]}\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "973ea6bf-95b0-4b14-853e-f4c57bc74ee5",
|
|
"sort_order": 2,
|
|
"rubric_text": "In the cash_exceptions_log spreadsheet, a new row must exist for the returned check with all required fields: original deposit date (03/12/2027) in column F, return date (03/25/2027) in column G, customer name Patricia Morris in column E, deal number 10091 in column D, check amount $2,300 in column J, and return reason code 'closed account' or 'account closed' in column L. MM/DD/YYYY formatting must be used for dates.",
|
|
"verifier_code": "from pathlib import Path\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available.\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\")) + list(Path(workspace_path).glob(\"*.xlsm\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Excel files found in workspace.\"}\n\n # Look for cash exception log files first, then fall back to all Excel files\n target_files = [f for f in xlsx_files if \"cash\" in f.name.lower() or \"exception\" in f.name.lower()]\n if not target_files:\n target_files = xlsx_files\n\n def date_to_strs(cell_value):\n \"\"\"Return multiple string representations for a date-like cell.\"\"\"\n results = []\n if cell_value is None:\n return [\"\"]\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n d = cell_value if isinstance(cell_value, datetime.date) else cell_value.date()\n results.append(d.strftime(\"%Y-%m-%d\"))\n results.append(d.strftime(\"%m/%d/%Y\"))\n results.append(d.strftime(\"%m/%d/%y\")) # MM/DD/YY\n results.append(f\"{d.month}/{d.day}/{d.year}\")\n results.append(f\"{d.month}/{d.day}/{str(d.year)[-2:]}\")\n results.append(d.strftime(\"%d-%b-%Y\").lower())\n results.append(d.strftime(\"%b %d\").lower())\n return results\n return [str(cell_value).lower().strip()]\n\n def check_date_match(cell_value, target_patterns):\n \"\"\"Check if a cell value matches any of the target date patterns.\"\"\"\n strs = date_to_strs(cell_value)\n for s in strs:\n for p in target_patterns:\n if p in s:\n return True\n # Also check plain string\n plain = str(cell_value).lower().strip() if cell_value is not None else \"\"\n for p in target_patterns:\n if p in plain:\n return True\n return False\n\n def cell_str(cell_value):\n if cell_value is None:\n return \"\"\n return str(cell_value).lower().strip()\n\n # Rubric says deposit date 03/12/2027 and return date 03/25/2027\n # Accept both MM/DD/YYYY and MM/DD/YY and other common formats\n deposit_date_patterns = [\"2027-03-12\", \"03/12/2027\", \"3/12/2027\", \"03/12/27\", \"3/12/27\", \"12-mar-2027\", \"12-mar\", \"mar 12\"]\n return_date_patterns = [\"2027-03-25\", \"03/25/2027\", \"3/25/2027\", \"03/25/27\", \"3/25/27\", \"25-mar-2027\", \"25-mar\", \"mar 25\"]\n\n best_match = None\n best_missing = None\n best_row_num = None\n best_found_details = None\n\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n except Exception:\n continue\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row_idx, row in enumerate(ws.iter_rows(), start=1):\n # Build row text for keyword searching\n row_strs = []\n for cell in row:\n row_strs.append(str(cell.value).lower() if cell.value is not None else \"\")\n row_text = \" \".join(row_strs)\n\n # Quick check: must have patricia/morris or 10091\n has_patricia = \"patricia\" in row_text and \"morris\" in row_text\n has_deal = \"10091\" in row_text\n\n if not (has_patricia or has_deal):\n continue\n\n missing = []\n found_details = []\n\n # Helper to get cell value by column index (1-indexed: A=1, D=4, etc.)\n def get_col(col_idx):\n \"\"\"Get cell value by 1-based column index from row tuple.\"\"\"\n if col_idx <= len(row):\n return row[col_idx - 1].value\n return None\n\n # Check customer name - Column E (index 5)\n col_e_val = cell_str(get_col(5))\n if \"patricia\" in col_e_val and \"morris\" in col_e_val:\n found_details.append(\"Customer Patricia Morris in col E\")\n elif \"patricia\" in row_text and \"morris\" in row_text:\n found_details.append(\"Customer Patricia Morris found (not necessarily col E)\")\n else:\n missing.append(\"Customer Patricia Morris in column E\")\n\n # Check deal number - Column D (index 4)\n col_d_val = cell_str(get_col(4))\n if \"10091\" in col_d_val:\n found_details.append(\"Deal number 10091 in col D\")\n elif \"10091\" in row_text:\n found_details.append(\"Deal number 10091 found (not necessarily col D)\")\n else:\n missing.append(\"Deal number 10091 in column D\")\n\n # Check amount - Column J (index 10)\n col_j_val = get_col(10)\n col_j_str = cell_str(col_j_val)\n col_j_numeric_match = False\n if isinstance(col_j_val, (int, float)):\n if abs(col_j_val - 2300) < 1:\n col_j_numeric_match = True\n if col_j_numeric_match or any(amt in col_j_str for amt in [\"2300\", \"2,300\"]):\n found_details.append(\"Check amount $2,300 in col J\")\n elif any(amt in row_text for amt in [\"2300\", \"2,300\"]):\n found_details.append(\"Check amount $2,300 found (not necessarily col J)\")\n else:\n missing.append(\"Check amount $2,300 in column J\")\n\n # Check reason - Column L (index 12)\n col_l_val = cell_str(get_col(12))\n if \"closed\" in col_l_val and (\"account\" in col_l_val or \"acct\" in col_l_val):\n found_details.append(\"Return reason 'closed account' in col L\")\n elif \"closed\" in col_l_val:\n found_details.append(\"Return reason with 'closed' in col L\")\n elif \"closed\" in row_text:\n found_details.append(\"Return reason with 'closed' found (not necessarily col L)\")\n else:\n missing.append(\"Return reason 'closed account' or 'account closed' in column L\")\n\n # Check deposit date (03/12/2027) - Column F (index 6)\n col_f_val = get_col(6)\n if check_date_match(col_f_val, deposit_date_patterns):\n found_details.append(\"Deposit date 03/12/2027 in col F\")\n else:\n found_deposit_anywhere = False\n for cell in row:\n if check_date_match(cell.value, deposit_date_patterns):\n found_deposit_anywhere = True\n break\n if found_deposit_anywhere:\n found_details.append(\"Deposit date 03/12/2027 found (not necessarily col F)\")\n else:\n missing.append(\"Original deposit date (03/12/2027) in column F\")\n\n # Check return date (03/25/2027) - Column G (index 7)\n col_g_val = get_col(7)\n if check_date_match(col_g_val, return_date_patterns):\n found_details.append(\"Return date 03/25/2027 in col G\")\n else:\n found_return_anywhere = False\n for cell in row:\n if check_date_match(cell.value, return_date_patterns):\n found_return_anywhere = True\n break\n if found_return_anywhere:\n found_details.append(\"Return date 03/25/2027 found (not necessarily col G)\")\n else:\n missing.append(\"Return date (03/25/2027) in column G\")\n\n if not missing:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found complete cash exception log entry in file '{fpath.name}', sheet '{sheet_name}', row {row_idx}: {'; '.join(found_details)}.\"\n }\n else:\n if best_missing is None or len(missing) < len(best_missing):\n best_missing = missing\n best_match = (fpath.name, sheet_name)\n best_row_num = row_idx\n best_found_details = found_details\n\n if best_match and best_missing and len(best_missing) <= 2:\n return {\n \"pass\": False,\n \"score\": 0.5,\n \"feedback\": f\"Found partial cash exception entry in '{best_match[0]}', sheet '{best_match[1]}', row {best_row_num}. Found: {'; '.join(best_found_details)}. Missing: {', '.join(best_missing)}\"\n }\n\n if best_match and best_missing:\n return {\n \"pass\": False,\n \"score\": 0.2,\n \"feedback\": f\"Found a row referencing Patricia Morris/10091 in '{best_match[0]}', sheet '{best_match[1]}', row {best_row_num} but missing {len(best_missing)} fields: {', '.join(best_missing)}. Found: {'; '.join(best_found_details)}\"\n }\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"No cash exception log entry found for Patricia Morris / Deal 10091 / $2,300 returned check with 'closed account' reason, deposit date 03/12/2027, and return date 03/25/2027. Searched files: \" + \", \".join(f.name for f in target_files)\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "cee17bf8-c3ac-46f6-a2ac-b984b0c55823",
|
|
"sort_order": 3,
|
|
"rubric_text": "In mailbox.json (Sent folder) or slack_data.json, Marcus Hale and Elena Brooks must both be notified about the returned check for Patricia Morris / Deal 10091 / $2,300 (via email or Slack).",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check mailbox or Slack.\"}\n \n marcus_notified = False\n elena_notified = False\n \n # Broad matching for email addresses / names\n marcus_ids = [\"marcus\", \"marcus.hale\", \"marcushale\", \"hale\"]\n elena_ids = [\"elena\", \"elena.brooks\", \"elenabrooks\", \"brooks\"]\n \n # Keywords that indicate the returned check context. Be loose: any two of these in the text is enough.\n returned_check_keywords = [\"returned\", \"return\", \"2,300\", \"2300\", \"patricia\", \"morris\", \"10091\", \"closed account\", \"nsf\", \"bounced\", \"check\", \"cheque\"]\n \n def text_has_returned_check_context(text):\n \"\"\"Return True if the text contains at least 2 of the returned-check keywords.\"\"\"\n matches = sum(1 for kw in returned_check_keywords if kw in text)\n return matches >= 2\n \n # ---- Check mailbox ----\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n bcc_addr = (email.get(\"bcc_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n combined_addr = to_addr + \" \" + cc_addr + \" \" + bcc_addr\n combined_text = body + \" \" + subject\n \n if not text_has_returned_check_context(combined_text):\n continue\n \n if any(m in combined_addr for m in marcus_ids):\n marcus_notified = True\n if any(e in combined_addr for e in elena_ids):\n elena_notified = True\n \n # ---- Check Slack ----\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n \n users = slack_data.get(\"users\", {})\n \n # Build sets of user IDs for Marcus and Elena\n marcus_user_ids = set()\n elena_user_ids = set()\n for uid, u in users.items():\n name_lower = (u.get(\"name\") or \"\").lower()\n real_name_lower = (u.get(\"real_name\") or \"\").lower()\n email_lower = (u.get(\"profile\", {}).get(\"email\") or \"\").lower()\n combined_user = name_lower + \" \" + real_name_lower + \" \" + email_lower\n if any(m in combined_user for m in [\"marcus\", \"hale\"]):\n marcus_user_ids.add(uid)\n if any(e in combined_user for e in [\"elena\", \"brooks\"]):\n elena_user_ids.add(uid)\n \n messages = slack_data.get(\"messages\", {})\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n text = (msg.get(\"text\") or \"\").lower()\n if not text_has_returned_check_context(text):\n continue\n \n sender = msg.get(\"user\", \"\")\n \n # Check if Marcus is the recipient (mentioned in text, or channel is a DM with Marcus)\n if any(m in text for m in [\"marcus\", \"hale\", \"@marcus\"]):\n marcus_notified = True\n # If sent by someone TO a DM channel where Marcus is a member, or sender is marcus\n if sender in marcus_user_ids:\n # Marcus sent it, so he's aware\n marcus_notified = True\n \n if any(e in text for e in [\"elena\", \"brooks\", \"@elena\"]):\n elena_notified = True\n if sender in elena_user_ids:\n elena_notified = True\n \n # Also check channel members if available\n channels = slack_data.get(\"channels\", {})\n channel_info = channels.get(channel_id, {})\n channel_members = channel_info.get(\"members\", [])\n # If it's a DM (is_im or is_mpim) and Marcus/Elena are members\n if channel_info.get(\"is_im\") or channel_info.get(\"is_mpim\") or channel_info.get(\"is_group\"):\n if any(uid in marcus_user_ids for uid in channel_members):\n marcus_notified = True\n if any(uid in elena_user_ids for uid in channel_members):\n elena_notified = True\n \n # Check if the DM channel name or id hints at the user\n channel_name = (channel_info.get(\"name\") or \"\").lower()\n if any(m in channel_name for m in [\"marcus\", \"hale\"]):\n marcus_notified = True\n if any(e in channel_name for e in [\"elena\", \"brooks\"]):\n elena_notified = True\n \n if marcus_notified and elena_notified:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Both Marcus Hale and Elena Brooks were notified about the returned check for Patricia Morris / Deal 10091 / $2,300.\"}\n elif marcus_notified:\n return {\"pass\": False, \"score\": 0.4, \"feedback\": \"Marcus Hale was notified but Elena Brooks was NOT notified about the returned check for Patricia Morris / Deal 10091 / $2,300. Both must be notified per SOP.\"}\n elif elena_notified:\n return {\"pass\": False, \"score\": 0.4, \"feedback\": \"Elena Brooks was notified but Marcus Hale was NOT notified about the returned check for Patricia Morris / Deal 10091 / $2,300. Both must be notified per SOP.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Neither Marcus Hale nor Elena Brooks were notified about the returned check for Patricia Morris / Deal 10091 / $2,300. No relevant sent emails or Slack messages found.\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "5df9f89c-6644-4a47-bf1f-4257422845a4",
|
|
"sort_order": 4,
|
|
"rubric_text": "In mailbox.json (Sent folder), a sent email to patricia2000@aol.com must exist notifying her about the returned check (referencing returned check, amount, or account issue).",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n returned_check_keywords = [\"returned\", \"return\", \"check\", \"payment\", \"2,300\", \"2300\", \"closed account\", \"account\", \"nsf\"]\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n \n if \"patricia2000@aol.com\" not in to_addr:\n continue\n \n combined_text = body + \" \" + subject\n has_returned_check = any(kw in combined_text for kw in returned_check_keywords)\n \n if has_returned_check:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found sent email to patricia2000@aol.com notifying about the returned check.\"}\n else:\n return {\"pass\": False, \"score\": 0.3, \"feedback\": \"Found email to patricia2000@aol.com but it does not appear to mention the returned check or payment issue.\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to patricia2000@aol.com notifying about the returned check.\"}",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "c9d032ac-b740-4c98-9ff4-97622e136a16",
|
|
"sort_order": 5,
|
|
"rubric_text": "In the Q1 2027 sheet of the deal_log spreadsheet, Column AG (Title Status) must contain 'Held - Returned Check'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available.\"}\n\n def _cell_str(v):\n return str(v).strip().lower() if v is not None else \"\"\n\n # Look for any xlsx/xlsm files that could be the deal_log spreadsheet\n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\")) + list(Path(workspace_path).glob(\"**/*.xlsm\"))\n\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No xlsx/xlsm files found in workspace: {workspace_path}\"}\n\n for fpath in xlsx_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n except Exception as e:\n continue\n\n # Look for a sheet that matches Q1 2027 (be lenient with naming)\n target_sheet = None\n for sheet_name in wb.sheetnames:\n sn_lower = sheet_name.strip().lower()\n if \"q1\" in sn_lower and \"2027\" in sn_lower:\n target_sheet = sheet_name\n break\n\n if target_sheet is None:\n # Also try sheets that just say \"q1 2027\" or similar\n for sheet_name in wb.sheetnames:\n sn_lower = sheet_name.strip().lower().replace(\" \", \"\")\n if \"q12027\" in sn_lower:\n target_sheet = sheet_name\n break\n\n if target_sheet is None:\n continue\n\n ws = wb[target_sheet]\n rows = list(ws.iter_rows())\n if not rows:\n continue\n\n # Column AG is the 33rd column (1-indexed), so 0-indexed it's 32\n ag_col_idx = 32 # 0-based index for column AG\n\n # Check every cell in column AG for the expected value\n found_values_ag = []\n for row in rows:\n if ag_col_idx < len(row):\n # Filter to only check deal #10091's row\n deal_num = row[0].value\n deal_str = str(deal_num).strip() if deal_num else \"\"\n if deal_str != \"10091\":\n continue\n cell_val = row[ag_col_idx].value\n cell_str = _cell_str(cell_val)\n found_values_ag.append(cell_val)\n\n # Be lenient: check if cell contains 'held' and 'returned check' (case insensitive)\n if \"held\" in cell_str and \"returned check\" in cell_str:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found 'Held - Returned Check' in Column AG (Title Status) of sheet '{target_sheet}' in file '{fpath.name}'. Cell value: '{cell_val}'.\"\n }\n\n # Secondary check: also look at header row to find a column labeled 'Title Status'\n # in case columns have been shifted\n header_row = rows[0] if rows else []\n title_status_col_idx = None\n for idx, cell in enumerate(header_row):\n val = _cell_str(cell.value)\n if \"title status\" in val or \"title_status\" in val or val == \"title\":\n title_status_col_idx = idx\n break\n\n if title_status_col_idx is not None and title_status_col_idx != ag_col_idx:\n for row in rows[1:]:\n if title_status_col_idx < len(row):\n # Filter to only check deal #10091's row\n deal_num = row[0].value\n deal_str = str(deal_num).strip() if deal_num else \"\"\n if deal_str != \"10091\":\n continue\n cell_val = row[title_status_col_idx].value\n cell_str = _cell_str(cell_val)\n if \"held\" in cell_str and \"returned check\" in cell_str:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found 'Held - Returned Check' in Title Status column (col index {title_status_col_idx}) of sheet '{target_sheet}' in file '{fpath.name}'. Cell value: '{cell_val}'.\"\n }\n\n # Collect AG column non-empty values for feedback\n non_empty_ag = [str(v) for v in found_values_ag if v is not None and str(v).strip() != \"\"]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Sheet '{target_sheet}' in file '{fpath.name}' was found, but Column AG (Title Status) does not contain 'Held - Returned Check'. Non-empty AG column values found: {non_empty_ag[:20]}\"\n }\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not find a 'Q1 2027' sheet in any spreadsheet file in workspace. Files checked: {[f.name for f in xlsx_files]}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "b2797c29-020a-42ff-b53e-34ce79041991",
|
|
"sort_order": 6,
|
|
"rubric_text": "In slack_data.json, a message in #acct-deal-posting must exist referencing Deal #10099 and a gross variance between the F&I recap and deal log.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n \"\"\"\n Rubric: In slack_data.json, a message in #acct-deal-posting must exist\n referencing Deal #10099 and a gross variance between the F&I recap and deal log.\n \"\"\"\n\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check slack_data.json.\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path.\"}\n\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n\n channels = slack_data.get(\"channels\", {})\n messages = slack_data.get(\"messages\", {})\n\n # Find the #acct-deal-posting channel(s)\n target_channel_ids = [\n cid for cid, c in channels.items()\n if \"acct-deal-posting\" in (c.get(\"name\") or \"\").lower()\n ]\n\n if not target_channel_ids:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No #acct-deal-posting channel found in slack_data.json channels.\"}\n\n def text_has_deal_ref(text):\n \"\"\"Check if text references Deal #10099 (loosely).\"\"\"\n t = text.lower()\n return \"10099\" in t\n\n def text_has_gross_variance_ref(text):\n \"\"\"Check if text references a gross variance between F&I recap and deal log.\n \n We look for:\n 1. Direct mention of 'gross' combined with variance/discrepancy language\n 2. Mentions of F&I recap and deal log with variance language\n 3. Any combination of gross, variance, discrepancy, mismatch, difference keywords\n 4. Dollar amounts with variance/discrepancy context\n \"\"\"\n t = text.lower()\n\n # Variance / discrepancy keywords\n variance_words = [\n \"variance\", \"discrepanc\", \"difference\", \"mismatch\",\n \"does not match\", \"doesn't match\", \"don't match\", \"do not match\",\n \"doesn\\u2019t match\", \"don\\u2019t match\",\n \"off by\", \"incorrect\", \"error\", \"issue\", \"flag\",\n \"inconsisten\", \"disagree\", \"conflict\", \"differ\",\n \"not matching\", \"not align\", \"misalign\", \"gap\"\n ]\n has_variance = any(w in t for w in variance_words)\n\n # Gross keyword\n has_gross = \"gross\" in t\n\n # F&I recap keywords\n fi_recap_words = [\"f&i\", \"fi recap\", \"f and i\", \"f & i\", \"recap\"]\n has_fi_recap = any(w in t for w in fi_recap_words)\n\n # Deal log keywords\n deal_log_words = [\"deal log\", \"deallog\", \"deal sheet\", \"deal record\"]\n has_deal_log = any(w in t for w in deal_log_words)\n\n has_dollar = bool(re.search(r'\\$[\\d,]+\\.?\\d*', t))\n has_number = bool(re.search(r'\\d{3,}', t))\n\n # Best case: gross + variance\n if has_gross and has_variance:\n return True\n\n # F&I recap + deal log + any variance/discrepancy\n if has_fi_recap and has_deal_log and has_variance:\n return True\n\n # F&I recap + deal log + gross (implies comparison)\n if has_fi_recap and has_deal_log and has_gross:\n return True\n\n # Gross + F&I recap or deal log (implies variance context)\n if has_gross and (has_fi_recap or has_deal_log):\n return True\n\n # Variance with dollar amount and gross\n if has_variance and has_dollar and has_gross:\n return True\n\n # Variance with F&I or deal log reference and dollar amount\n if has_variance and (has_fi_recap or has_deal_log) and has_dollar:\n return True\n\n # Loose: gross + dollar amount + any context word\n context_words = [\"profit\", \"total\", \"front\", \"back\", \"posting\", \"review\"]\n has_context = any(w in t for w in context_words)\n if has_gross and has_dollar and has_context:\n return True\n\n # Very loose: gross + variance language even without dollar amounts\n if has_gross and has_variance:\n return True\n\n # Loose: if recap and deal log are both mentioned with any numbers, that implies a comparison\n if has_fi_recap and has_deal_log and has_number:\n return True\n\n # Accept known specific amounts from the deal that indicate gross figures\n known_amounts = [\"8,224\", \"8224\", \"8,220\", \"8220\", \"6,700\", \"6700\", \"7,500\", \"7500\"]\n if any(amt in t for amt in known_amounts) and has_gross:\n return True\n\n return False\n\n # Search in target channels - strict matching first\n for cid in target_channel_ids:\n channel_msgs = messages.get(cid, [])\n for msg in channel_msgs:\n text = msg.get(\"text\") or \"\"\n if text_has_deal_ref(text) and text_has_gross_variance_ref(text):\n channel_name = channels.get(cid, {}).get(\"name\", cid)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found Slack message in #{channel_name} referencing Deal #10099 and a gross variance between F&I recap and deal log. Message snippet: {text[:300]}\"\n }\n\n # Looser pass: if message references 10099 and has 'gross' plus any number\n for cid in target_channel_ids:\n channel_msgs = messages.get(cid, [])\n for msg in channel_msgs:\n text = msg.get(\"text\") or \"\"\n t = text.lower()\n if text_has_deal_ref(text) and \"gross\" in t and re.search(r'\\d{3,}', text):\n channel_name = channels.get(cid, {}).get(\"name\", cid)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found Slack message in #{channel_name} referencing Deal #10099 with gross and numeric values (implied variance). Message snippet: {text[:300]}\"\n }\n\n # Even looser: 10099 + (variance OR discrepancy OR mismatch) + any number\n for cid in target_channel_ids:\n channel_msgs = messages.get(cid, [])\n for msg in channel_msgs:\n text = msg.get(\"text\") or \"\"\n t = text.lower()\n variance_words = [\"variance\", \"discrepanc\", \"mismatch\", \"doesn't match\", \"does not match\", \"difference\", \"incorrect\", \"flag\", \"issue\", \"error\"]\n has_var = any(w in t for w in variance_words)\n if text_has_deal_ref(text) and has_var and re.search(r'\\d{3,}', text):\n channel_name = channels.get(cid, {}).get(\"name\", cid)\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found Slack message in #{channel_name} referencing Deal #10099 with variance language and numeric values. Message snippet: {text[:300]}\"\n }\n\n # Provide debugging info\n found_deal_anywhere = False\n found_gross_anywhere = False\n found_variance_anywhere = False\n total_target_msgs = 0\n all_texts = []\n for cid in target_channel_ids:\n channel_msgs = messages.get(cid, [])\n total_target_msgs += len(channel_msgs)\n for msg in channel_msgs:\n text = msg.get(\"text\") or \"\"\n if text.strip():\n all_texts.append(text[:200])\n if text_has_deal_ref(text):\n found_deal_anywhere = True\n if \"gross\" in text.lower():\n found_gross_anywhere = True\n variance_words = [\"variance\", \"discrepanc\", \"mismatch\", \"difference\"]\n if any(w in text.lower() for w in variance_words):\n found_variance_anywhere = True\n\n missing = []\n if not found_deal_anywhere:\n missing.append(\"Deal #10099 reference\")\n if not found_gross_anywhere:\n missing.append(\"gross reference\")\n if not found_variance_anywhere:\n missing.append(\"variance/discrepancy language\")\n if found_deal_anywhere and (found_gross_anywhere or found_variance_anywhere):\n missing.append(\"required elements exist but not combined properly in the same message\")\n\n channel_names = [channels.get(cid, {}).get(\"name\", cid) for cid in target_channel_ids]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No matching message found in #acct-deal-posting ({', '.join(channel_names)}) referencing Deal #10099 AND a gross variance between F&I recap and deal log. Scanned {total_target_msgs} messages. Missing: {', '.join(missing)}. Sample texts: {all_texts[:5]}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "d0da5a96-d087-48a7-9d36-6da54673deb9",
|
|
"sort_order": 7,
|
|
"rubric_text": "In mailbox.json (Sent folder), a reply email to Marcus Bell's 'New Deal Jacket' email must exist communicating that Deal #10099 has a gross variance requiring Marcus Hale review and cannot be posted yet. ",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n all_emails = data.get(\"emails\", [])\n \n # Find Marcus Bell's 'New Deal Jacket' email (the original inbound email)\n marcus_bell_email = None\n for email in all_emails:\n from_addr = (email.get(\"from_addr\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n folder = (email.get(\"folder\") or \"\").lower()\n if (\"bell\" in from_addr or \"marcus\" in from_addr) and (\"deal jacket\" in subject or \"new deal\" in subject):\n if folder != \"sent\":\n marcus_bell_email = email\n break\n \n # Find sent emails\n sent_emails = [e for e in all_emails if (e.get(\"folder\") or \"\").lower() == \"sent\"]\n \n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent emails found in mailbox.\"}\n \n best_match = None\n best_score = 0.0\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n in_reply_to = email.get(\"in_reply_to\") or \"\"\n combined_text = body + \" \" + subject\n all_recipients = to_addr + \" \" + cc_addr\n \n # Check if this email is related to Marcus Bell / the deal jacket topic\n is_to_bell = \"bell\" in all_recipients\n is_reply_to_bell = False\n if marcus_bell_email:\n original_message_id = marcus_bell_email.get(\"message_id\") or \"\"\n if in_reply_to and original_message_id and in_reply_to == original_message_id:\n is_reply_to_bell = True\n \n is_about_deal_jacket = \"deal jacket\" in combined_text or \"new deal\" in combined_text or \"10099\" in combined_text\n \n # Must be either a reply to Bell, sent to Bell, or at least mention Bell and the deal\n is_bell_related = is_reply_to_bell or is_to_bell or (\"bell\" in combined_text and is_about_deal_jacket)\n \n if not is_bell_related:\n continue\n \n # --- Rubric core: gross variance, Marcus Hale review, cannot be posted, Deal #10099 ---\n \n # Check for ANY mention of variance / discrepancy / mismatch (loosely)\n has_variance_language = any(kw in combined_text for kw in [\n \"variance\", \"discrepancy\", \"difference\", \"mismatch\",\n \"does not match\", \"doesn't match\", \"don't match\", \"do not match\",\n \"inconsisten\", \"differ\", \"off by\", \"gross\"\n ])\n \n # Check for specific $4 variance or the two amounts\n has_4_variance = any(kw in combined_text for kw in [\"$4\", \"4 dollar\", \"four dollar\", \"$4.00\", \"4.00 variance\", \"4 variance\"])\n has_4_variance = has_4_variance or bool(re.search(r'\\$4(?:\\b|\\.00)', combined_text))\n \n has_8220 = any(kw in combined_text for kw in [\"8,220\", \"8220\", \"$8,220\", \"$8220\"])\n has_8224 = any(kw in combined_text for kw in [\"8,224\", \"8224\", \"$8,224\", \"$8224\"])\n has_both_amounts = has_8220 and has_8224\n \n has_explicit_variance_reference = has_4_variance or has_both_amounts\n \n # Check cannot-post / hold language\n has_cannot_post = any(kw in combined_text for kw in [\n \"cannot post\", \"can't post\", \"cannot be posted\", \"not post\",\n \"hold\", \"pending\", \"unable to post\", \"won't post\", \"will not post\",\n \"not ready\", \"on hold\", \"cannot finalize\", \"can't finalize\",\n \"cannot proceed\", \"can't proceed\", \"not yet\", \"before post\",\n \"prior to post\", \"needs to be resolved\", \"need to resolve\",\n \"must be resolved\", \"before we can\", \"until\"\n ])\n \n # References Marcus Hale / review / escalation\n has_hale = \"hale\" in combined_text\n has_review = any(kw in combined_text for kw in [\"review\", \"approv\", \"sign off\", \"sign-off\", \"escalat\", \"flag\"])\n has_hale_review = has_hale or has_review\n \n # References deal 10099\n has_deal_ref = \"10099\" in combined_text\n \n # --- Scoring ---\n # The rubric requires:\n # 1. Deal #10099 has a gross variance\n # 2. Requires Marcus Hale review\n # 3. Cannot be posted yet\n \n score = 0.0\n \n # Must mention variance in some way (gross variance)\n if has_variance_language or has_explicit_variance_reference:\n score = 0.4\n else:\n # No mention of variance at all - skip\n continue\n \n # Deal reference\n if has_deal_ref:\n score += 0.15\n \n # Marcus Hale review\n if has_hale_review:\n score += 0.2\n \n # Cannot be posted\n if has_cannot_post:\n score += 0.15\n \n # Bonus for explicit variance amount\n if has_explicit_variance_reference:\n score += 0.1\n \n # Cap at 1.0\n score = min(score, 1.0)\n \n # Bonus for being a direct reply\n if is_reply_to_bell:\n score = min(score + 0.02, 1.0)\n \n # Full marks path: variance + cannot post + hale review + deal ref\n if has_variance_language and has_cannot_post and has_hale_review and has_deal_ref:\n score = max(score, 1.0)\n \n if has_explicit_variance_reference and (has_cannot_post or has_hale_review):\n score = max(score, 1.0)\n \n if score > best_score:\n best_score = score\n best_match = email\n \n # Passing threshold: 0.7\n if best_match and best_score >= 0.7:\n body_preview = (best_match.get(\"body_text\") or \"\")[:200]\n return {\n \"pass\": True,\n \"score\": best_score,\n \"feedback\": f\"Found sent email to/about Marcus Bell communicating Deal #10099 gross variance requiring review and cannot be posted yet. Score: {best_score}. Preview: {body_preview}\"\n }\n \n # Provide detailed failure feedback\n bell_emails = [e for e in sent_emails if \"bell\" in (e.get(\"to_addr\") or \"\").lower() or \"bell\" in (e.get(\"body_text\") or \"\").lower() or \"bell\" in (e.get(\"subject\") or \"\").lower()]\n if bell_emails:\n subjects = [e.get(\"subject\", \"(no subject)\") for e in bell_emails]\n bodies_preview = [(e.get(\"body_text\") or \"\")[:150] for e in bell_emails]\n return {\n \"pass\": False,\n \"score\": best_score,\n \"feedback\": f\"Found {len(bell_emails)} sent email(s) mentioning Bell (subjects: {subjects}), but none adequately communicated that Deal #10099 has a gross variance requiring Marcus Hale review and cannot be posted yet. Best score: {best_score}. Previews: {bodies_preview}\"\n }\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"No reply found to Marcus Bell's 'New Deal Jacket' email. Expected a sent email communicating that Deal #10099 has a gross variance requiring Marcus Hale review and cannot be posted yet.\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "d483f22b-bd8b-4ae3-b44d-e2acd7d831b9",
|
|
"sort_order": 8,
|
|
"rubric_text": "In the Q1 2027 sheet of the deal_log spreadsheet, the figures for Deal #10099 must be unchanged: cell AC27 must contain exactly $2,161, cell AD27 must contain exactly $6,059 and cell AE27 must contain exactly $8,220.",
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available.\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\")) + list(Path(workspace_path).glob(\"*.xlsm\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Excel files found in workspace.\"}\n\n deal_log_files = [f for f in xlsx_files if \"deal\" in f.name.lower() and \"log\" in f.name.lower()]\n if not deal_log_files:\n deal_log_files = xlsx_files\n\n def parse_dollar(val):\n \"\"\"Try to parse a cell value as a numeric dollar amount.\"\"\"\n if val is None:\n return None\n if isinstance(val, (int, float)):\n return round(float(val), 2)\n s = str(val).strip().replace(\"$\", \"\").replace(\",\", \"\").strip()\n try:\n return round(float(s), 2)\n except (ValueError, TypeError):\n return None\n\n for fpath in deal_log_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n except Exception:\n continue\n\n # Look for a sheet that matches \"Q1 2027\" loosely\n target_sheet = None\n for sheet_name in wb.sheetnames:\n sn_lower = sheet_name.lower().replace(\" \", \"\")\n if \"q1\" in sn_lower and \"2027\" in sn_lower:\n target_sheet = sheet_name\n break\n\n if target_sheet is None:\n sheets_to_check = wb.sheetnames\n else:\n sheets_to_check = [target_sheet]\n\n for sheet_name in sheets_to_check:\n ws = wb[sheet_name]\n\n # The rubric says row 27. First verify deal 10099 is there.\n row_num = 27\n\n row_has_10099 = False\n for cell in ws[row_num]:\n if cell.value is not None and \"10099\" in str(cell.value):\n row_has_10099 = True\n break\n\n if not row_has_10099:\n # Try to find 10099 in other rows of this sheet\n found_row = None\n for row in ws.iter_rows():\n for cell in row:\n if cell.value is not None and \"10099\" in str(cell.value):\n found_row = cell.row\n break\n if found_row:\n break\n if found_row:\n row_num = found_row\n else:\n continue\n\n # AC = column 29, AD = column 30, AE = column 31\n val_ac = ws.cell(row=row_num, column=29).value\n val_ad = ws.cell(row=row_num, column=30).value\n val_ae = ws.cell(row=row_num, column=31).value\n\n num_ac = parse_dollar(val_ac)\n num_ad = parse_dollar(val_ad)\n num_ae = parse_dollar(val_ae)\n\n issues = []\n passes = []\n\n # Check AC = $2,161 (tolerance ±1.5)\n if num_ac is not None and abs(num_ac - 2161.0) < 1.5:\n passes.append(f\"AC{row_num} = ${num_ac} (expected $2,161) ✓\")\n else:\n issues.append(f\"AC{row_num} = {val_ac} (parsed: {num_ac}), expected $2,161\")\n\n # Check AD = $6,059 (tolerance ±1.5)\n if num_ad is not None and abs(num_ad - 6059.0) < 1.5:\n passes.append(f\"AD{row_num} = ${num_ad} (expected $6,059) ✓\")\n else:\n issues.append(f\"AD{row_num} = {val_ad} (parsed: {num_ad}), expected $6,059\")\n\n # Check AE = $8,220 (tolerance ±1.5)\n if num_ae is not None and abs(num_ae - 8220.0) < 1.5:\n passes.append(f\"AE{row_num} = ${num_ae} (expected $8,220) ✓\")\n else:\n issues.append(f\"AE{row_num} = {val_ae} (parsed: {num_ae}), expected $8,220\")\n\n if not issues:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Deal #10099 figures in sheet '{sheet_name}' of '{fpath.name}' are unchanged: {'; '.join(passes)}\"\n }\n else:\n score = len(passes) / (len(passes) + len(issues))\n pass_result = score >= 1.0\n return {\n \"pass\": pass_result,\n \"score\": round(score, 2),\n \"feedback\": f\"Deal #10099 in sheet '{sheet_name}' of '{fpath.name}': PASSED: {passes}; ISSUES: {issues}\"\n }\n\n return {\"pass\": True, \"score\": 0.8, \"feedback\": \"Deal #10099 not found in any deal_log sheet - cannot verify figures were unchanged, but no alteration detected.\"}\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "rubric_1776334609340",
|
|
"sort_order": 9,
|
|
"rubric_text": "No sent email replies or Slack messages must be sent regarding the number of entries in the cash exception log, from the already-read email with subject 'cash exception log' sent from elena.brooks@sunshineandsetauto.com regarding number of entries in the log. ",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n feedback_parts = []\n found_violation = False\n\n # Check external services (email)\n if external_services_path is not None:\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, 'r') as f:\n mailbox_data = json.load(f)\n emails = mailbox_data.get(\"emails\", [])\n # Look for sent emails that reference 'cash exception log' in subject or body\n for email in emails:\n folder = str(email.get(\"folder\", \"\")).lower()\n subject = str(email.get(\"subject\", \"\")).lower()\n body = str(email.get(\"body_text\", \"\") or email.get(\"body\", \"\")).lower()\n to_addr = str(email.get(\"to_addr\", \"\") or email.get(\"to\", \"\")).lower()\n if folder == \"sent\":\n # Check if the subject references 'cash exception log'\n if \"cash exception log\" in subject:\n found_violation = True\n feedback_parts.append(\n f\"VIOLATION: Found sent email with subject '{email.get('subject')}' \"\n f\"to '{email.get('to_addr', email.get('to', 'unknown'))}' referencing 'cash exception log'.\"\n )\n # Also check body referencing 'cash exception log' and 'entries' / 'number of entries'\n elif \"cash exception log\" in body and (\"entries\" in body or \"number of\" in body):\n found_violation = True\n feedback_parts.append(\n f\"VIOLATION: Found sent email to '{email.get('to_addr', email.get('to', 'unknown'))}' \"\n f\"with body referencing 'cash exception log' and entries.\"\n )\n if not found_violation:\n feedback_parts.append(\"No sent emails found referencing the 'cash exception log' topic. Good.\")\n else:\n feedback_parts.append(\"mailbox.json not found; cannot verify sent emails.\")\n else:\n feedback_parts.append(\"No external_services_path provided; cannot verify sent emails directly.\")\n\n # Check for Slack messages\n if external_services_path is not None:\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, 'r') as f:\n slack_data = json.load(f)\n # Correctly iterate over slack messages per the documented format:\n # messages is a dict keyed by channel ID -> list of message objects\n slack_messages_dict = slack_data.get(\"messages\", {})\n all_messages = []\n if isinstance(slack_messages_dict, dict):\n for channel_id, channel_msgs in slack_messages_dict.items():\n if isinstance(channel_msgs, list):\n all_messages.extend(channel_msgs)\n # Also check if there's a flat list (fallback for alternative formats)\n elif isinstance(slack_messages_dict, list):\n all_messages.extend(slack_messages_dict)\n\n slack_violation_found = False\n for msg in all_messages:\n msg_text = str(msg.get(\"text\", \"\") or msg.get(\"message\", \"\") or msg.get(\"body\", \"\")).lower()\n if \"cash exception log\" in msg_text and (\"entries\" in msg_text or \"number of\" in msg_text):\n found_violation = True\n slack_violation_found = True\n feedback_parts.append(\n f\"VIOLATION: Found Slack message referencing 'cash exception log' and entries: {msg.get('text', msg)}\"\n )\n if not slack_violation_found:\n feedback_parts.append(\"No Slack messages found referencing 'cash exception log' entries. Good.\")\n else:\n feedback_parts.append(\"No slack_data.json found; no Slack violations detected.\")\n\n if found_violation:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"FAIL: Agent sent email replies or Slack messages about the already-read 'cash exception log' email. \" + \" | \".join(feedback_parts)\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No sent email replies or Slack messages were found regarding the already-read 'cash exception log' email. \" + \" | \".join(feedback_parts)\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776334854994",
|
|
"sort_order": 10,
|
|
"rubric_text": "No new sent emails or Slack messages must be sent regarding or in reply to the email with subject 'New vehicles total'.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n feedback_parts = []\n found_violation = False\n subject_pattern = re.compile(r'new\\s*vehicles?\\s*total', re.IGNORECASE)\n\n # Check sent emails in mailbox.json\n if external_services_path:\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n try:\n with open(mailbox_path, 'r') as f:\n mailbox_data = json.load(f)\n emails = mailbox_data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for email in sent_emails:\n subj = email.get(\"subject\", \"\")\n body = email.get(\"body_text\", \"\") or email.get(\"body\", \"\")\n # Check if the subject line or body references 'New vehicles total'\n if subject_pattern.search(str(subj)) or subject_pattern.search(str(body)):\n found_violation = True\n feedback_parts.append(\n f\"Found sent email (id={email.get('email_id')}) with subject '{subj}' \"\n f\"that references 'New vehicles total'.\"\n )\n if not found_violation:\n feedback_parts.append(f\"Checked {len(sent_emails)} sent email(s); none reference 'New vehicles total'.\")\n except Exception as e:\n feedback_parts.append(f\"Error reading mailbox.json: {e}\")\n else:\n feedback_parts.append(\"No mailbox.json found; assuming no sent emails.\")\n else:\n feedback_parts.append(\"No external_services_path provided; cannot check sent emails directly. Assuming pass for email check.\")\n\n # Check Slack messages\n if external_services_path:\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n try:\n with open(slack_path, 'r') as f:\n slack_data = json.load(f)\n messages = []\n if isinstance(slack_data, dict):\n # Handle standard slack_data.json format: messages is dict of channel_id -> list of messages\n msgs_section = slack_data.get(\"messages\", {})\n if isinstance(msgs_section, dict):\n for channel_id, channel_msgs in msgs_section.items():\n if isinstance(channel_msgs, list):\n messages.extend(channel_msgs)\n # Also check other top-level keys that might contain message lists\n for key, value in slack_data.items():\n if key == \"messages\":\n continue\n if isinstance(value, list):\n for item in value:\n if isinstance(item, dict) and (\"text\" in item or \"message\" in item or \"content\" in item):\n messages.append(item)\n elif isinstance(slack_data, list):\n messages = slack_data\n\n slack_violation_found = False\n for msg in messages:\n if isinstance(msg, dict):\n text = str(msg.get(\"text\", \"\") or msg.get(\"message\", \"\") or msg.get(\"body\", \"\") or msg.get(\"content\", \"\"))\n if subject_pattern.search(text):\n found_violation = True\n slack_violation_found = True\n feedback_parts.append(\n f\"Found Slack message referencing 'New vehicles total': {text[:200]}\"\n )\n if not slack_violation_found:\n feedback_parts.append(f\"Checked {len(messages)} Slack message(s); none reference 'New vehicles total'.\")\n except Exception as e:\n feedback_parts.append(f\"Error reading slack_data.json: {e}\")\n else:\n feedback_parts.append(\"No slack_data.json found; assuming no Slack messages sent.\")\n else:\n feedback_parts.append(\"No external_services_path provided; cannot check Slack messages directly. Assuming pass for Slack check.\")\n\n if found_violation:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": \"FAIL: Found sent emails or Slack messages regarding 'New vehicles total'. \" + \" | \".join(feedback_parts)\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: No sent emails or Slack messages were found regarding 'New vehicles total'. \" + \" | \".join(feedback_parts)\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
},
|
|
{
|
|
"id": "rubric_1776778125459",
|
|
"sort_order": 11,
|
|
"rubric_text": "A sent email addressed to Marcus Hale (marcus.hale@sunshineandsetauto.com), or a Slack message tagging Marcus Hale, or a direct Slack message to Marcus Hale must exist, notifying him about Deal #10099 and a gross variance.",
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n findings = []\n found_email = False\n found_slack = False\n deal_mentioned = False\n variance_mentioned = False\n\n # Check sent emails\n if external_services_path:\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n try:\n with open(mailbox_path, \"r\") as f:\n mailbox = json.load(f)\n emails = mailbox.get(\"emails\", [])\n for email in emails:\n folder = (email.get(\"folder\") or \"\").lower()\n if folder != \"sent\":\n continue\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n combined = subject + \" \" + body\n all_recipients = to_addr + \" \" + cc_addr\n # Check if addressed to Marcus Hale\n if \"marcus.hale@sunshineandsetauto.com\" in all_recipients or \"marcus.hale\" in all_recipients or \"marcus hale\" in all_recipients:\n found_email = True\n findings.append(f\"Found sent email to: {email.get('to_addr')}, cc: {email.get('cc_addr', '')}, subject: {email.get('subject')}\")\n # Check for deal 10099\n if \"10099\" in combined:\n deal_mentioned = True\n # Check for gross variance mention - be very generous\n variance_keywords = [\"variance\", \"discrepancy\", \"mismatch\", \"difference\", \"gross\",\n \"does not match\", \"doesn't match\", \"doesn't match\",\n \"incorrect\", \"exception\", \"inconsisten\", \"error\",\n \"issue\", \"conflict\", \"off by\", \"posting\",\n \"not matching\", \"do not match\", \"don't match\",\n \"don't match\", \"deviation\", \"discrepanc\"]\n for kw in variance_keywords:\n if kw in combined:\n variance_mentioned = True\n break\n except Exception as e:\n findings.append(f\"Error reading mailbox: {e}\")\n else:\n findings.append(\"mailbox.json not found at external_services_path\")\n\n # Check Slack messages via slack_data.json\n if external_services_path:\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n try:\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n messages_dict = slack_data.get(\"messages\", {})\n for channel_id, msg_list in messages_dict.items():\n if not isinstance(msg_list, list):\n continue\n for msg in msg_list:\n text = (msg.get(\"text\") or \"\").lower()\n # Check for Marcus Hale mention\n if \"marcus\" in text and (\"hale\" in text or \"marcus.hale\" in text or \"@marcus\" in text):\n found_slack = True\n findings.append(f\"Found Slack message mentioning Marcus Hale in channel {channel_id}: {text[:100]}\")\n if \"10099\" in text:\n deal_mentioned = True\n variance_keywords = [\"variance\", \"discrepancy\", \"mismatch\", \"difference\", \"gross\",\n \"does not match\", \"doesn't match\", \"doesn't match\",\n \"incorrect\", \"exception\", \"inconsisten\", \"error\",\n \"issue\", \"conflict\", \"off by\", \"posting\",\n \"not matching\", \"do not match\", \"don't match\",\n \"don't match\", \"deviation\"]\n for kw in variance_keywords:\n if kw in text:\n variance_mentioned = True\n break\n # Also check if message tags a Marcus user by user ID\n user_id = msg.get(\"user\", \"\")\n users = slack_data.get(\"users\", {})\n # Check if any user mention in text resolves to Marcus Hale\n for uid, uinfo in users.items():\n uname = (uinfo.get(\"real_name\") or \"\").lower()\n if \"marcus\" in uname and \"hale\" in uname:\n # Check if this user is mentioned or is the recipient of a DM\n if f\"<@{uid}>\" in (msg.get(\"text\") or \"\").lower() or f\"<@{uid.lower()}>\" in (msg.get(\"text\") or \"\").lower() or uid.lower() in text:\n found_slack = True\n findings.append(f\"Found Slack message tagging Marcus Hale user {uid}\")\n if \"10099\" in text:\n deal_mentioned = True\n for kw in [\"variance\", \"discrepancy\", \"mismatch\", \"difference\", \"gross\",\n \"does not match\", \"doesn't match\", \"incorrect\", \"exception\",\n \"issue\", \"posting\", \"error\"]:\n if kw in text:\n variance_mentioned = True\n break\n except Exception as e:\n findings.append(f\"Error reading slack_data.json: {e}\")\n\n # Also check workspace for any slack-related files as fallback\n ws = Path(workspace_path)\n slack_paths = []\n for pattern in [\"**/*slack*\", \"**/*message*\", \"**/*chat*\"]:\n slack_paths.extend(ws.glob(pattern))\n\n for sp in slack_paths:\n try:\n with open(sp, \"r\") as f:\n content = f.read().lower()\n if \"marcus\" in content and (\"hale\" in content or \"marcus.hale\" in content or \"@marcus\" in content):\n found_slack = True\n findings.append(f\"Found Slack file with Marcus Hale mention: {sp.name}\")\n if \"10099\" in content:\n deal_mentioned = True\n variance_keywords = [\"variance\", \"discrepancy\", \"mismatch\", \"difference\", \"gross\",\n \"does not match\", \"doesn't match\", \"incorrect\", \"exception\",\n \"issue\", \"posting\", \"error\"]\n for kw in variance_keywords:\n if kw in content:\n variance_mentioned = True\n break\n except Exception:\n pass\n\n # Determine pass/fail\n contacted_marcus = found_email or found_slack\n if not contacted_marcus:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: No sent email or Slack message to/about Marcus Hale was found. Details: {'; '.join(findings) if findings else 'No relevant communications found.'}\"\n }\n\n if not deal_mentioned:\n return {\n \"pass\": False,\n \"score\": 0.3,\n \"feedback\": f\"FAIL: Marcus Hale was contacted but Deal #10099 was not mentioned. Details: {'; '.join(findings)}\"\n }\n\n if not variance_mentioned:\n return {\n \"pass\": False,\n \"score\": 0.5,\n \"feedback\": f\"FAIL: Marcus Hale was contacted about Deal #10099, but no mention of a gross variance or related issue was found. Details: {'; '.join(findings)}\"\n }\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: Marcus Hale was notified about Deal #10099 and a gross variance. Details: {'; '.join(findings)}\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "new_p3_hold_notification",
|
|
"sort_order": 12,
|
|
"rubric_text": "In slack_data.json, a message in #acct-titles-billing (channel C003) must exist referencing Deal #10091 and a title hold or returned-check hold, indicating that pending title work must be suspended or halted. The message must include the deal number (10091) and reference the returned check, hold status, or need to suspend title/registration work.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check slack_data.json.\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n\n messages = slack_data.get(\"messages\", {})\n c003_msgs = messages.get(\"C003\", [])\n\n if not c003_msgs:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in channel C003 (#acct-titles-billing).\"}\n\n # The seeded Priya Bennett message has ts 1806044400.000100 — skip it\n seeded_ts = \"1806044400.000100\"\n\n hold_keywords = [\n \"hold\", \"held\", \"returned check\", \"returned\", \"suspend\",\n \"halt\", \"stop\", \"block\", \"pending\", \"do not release\",\n \"do not submit\", \"title\"\n ]\n\n for msg in c003_msgs:\n ts = msg.get(\"ts\", \"\")\n if ts == seeded_ts:\n continue\n text = (msg.get(\"text\") or \"\").lower()\n\n has_deal = \"10091\" in text\n if not has_deal:\n continue\n\n keyword_count = sum(1 for kw in hold_keywords if kw in text)\n\n if keyword_count >= 2:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found message in #acct-titles-billing (C003) referencing Deal #10091 with hold/returned-check context ({keyword_count} keywords matched). Message: {text[:300]}\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.3,\n \"feedback\": f\"Found message in C003 mentioning 10091 but lacking sufficient hold/returned-check context (only {keyword_count} keyword(s) matched; need at least 2). Message: {text[:300]}\"\n }\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No message in #acct-titles-billing (C003) references Deal #10091. Found {len(c003_msgs)} message(s) in channel.\"\n }\n",
|
|
"criterion_type": "expected_output"
|
|
},
|
|
{
|
|
"id": "new_prior_quarter_check",
|
|
"sort_order": 13,
|
|
"rubric_text": "Agent correctly applied the standard $1,000 escalation threshold (not the reduced $500 prior-quarter threshold) because Deal #10091's delivery date (March 13, 2027) falls in Q1 2027 — the current quarter, not a prior quarter. Agent did NOT send an additional notification to Talia Nguyen.",
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n \"\"\"\n Check that the agent did NOT incorrectly apply the prior-quarter exception.\n The prior-quarter exception (reduced $500 threshold + Talia Nguyen notification)\n does NOT apply because Deal #10091's delivery date (March 13, 2027) is in Q1 2027,\n which is the current quarter (not a prior quarter).\n \n PASS if: No notification was sent to Talia Nguyen about the returned check.\n FAIL if: Agent sent a notification to Talia Nguyen about the returned check for Deal #10091.\n \"\"\"\n if not external_services_path:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external_services_path provided; cannot check for incorrect Talia Nguyen notification. Assuming correct behavior.\"}\n \n talia_notified = False\n evidence = []\n \n # Check mailbox for emails to Talia Nguyen about the returned check\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n talia_ids = [\"talia\", \"nguyen\", \"talia.nguyen\"]\n returned_check_keywords = [\"returned\", \"return\", \"check\", \"10091\", \"morris\", \"2,300\", \"2300\", \"closed account\", \"nsf\", \"bounced\", \"title-hold\", \"title hold\", \"priority\"]\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n bcc_addr = (email.get(\"bcc_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n combined_addr = to_addr + \" \" + cc_addr + \" \" + bcc_addr\n combined_text = body + \" \" + subject\n \n # Check if sent to Talia Nguyen\n is_to_talia = any(t in combined_addr for t in talia_ids)\n if not is_to_talia:\n continue\n \n # Check if it's about the returned check\n keyword_matches = sum(1 for kw in returned_check_keywords if kw in combined_text)\n if keyword_matches >= 2:\n talia_notified = True\n evidence.append(f\"Email to Talia Nguyen (to: {to_addr}) about returned check. Subject: {email.get('subject', '(none)')}\")\n \n # Check Slack for messages mentioning Talia Nguyen about the returned check\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n \n messages = slack_data.get(\"messages\", {})\n users = slack_data.get(\"users\", {})\n \n # Find Talia Nguyen user IDs\n talia_user_ids = set()\n for uid, u in users.items():\n name_lower = (u.get(\"name\") or \"\").lower()\n real_name_lower = (u.get(\"real_name\") or \"\").lower()\n email_lower = (u.get(\"profile\", {}).get(\"email\") or \"\").lower()\n combined_user = name_lower + \" \" + real_name_lower + \" \" + email_lower\n if \"talia\" in combined_user or \"nguyen\" in combined_user:\n talia_user_ids.add(uid)\n \n returned_check_keywords = [\"returned\", \"return\", \"check\", \"10091\", \"morris\", \"2,300\", \"2300\", \"closed account\", \"nsf\", \"bounced\", \"title-hold\", \"title hold\", \"priority\"]\n \n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n text = (msg.get(\"text\") or \"\").lower()\n \n # Check if Talia is mentioned\n mentions_talia = \"talia\" in text or \"nguyen\" in text or any(f\"<@{uid}>\" in text for uid in talia_user_ids)\n if not mentions_talia:\n continue\n \n # Check if it's about the returned check\n keyword_matches = sum(1 for kw in returned_check_keywords if kw in text)\n if keyword_matches >= 2:\n talia_notified = True\n evidence.append(f\"Slack message mentioning Talia Nguyen in channel {channel_id} about returned check: {text[:200]}\")\n \n if talia_notified:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: Agent incorrectly notified Talia Nguyen about the returned check for Deal #10091. The prior-quarter exception does not apply because Deal #10091's delivery date (March 13, 2027) is in Q1 2027, the current quarter. Evidence: {'; '.join(evidence)}\"\n }\n else:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": \"PASS: Agent correctly did NOT notify Talia Nguyen about the returned check. The prior-quarter exception does not apply to Deal #10091 (delivery date March 13, 2027 = Q1 2027 = current quarter).\"\n }\n",
|
|
"criterion_type": "incorrect_behavior"
|
|
}
|
|
]
|