101 lines
88 KiB
JSON
101 lines
88 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"id": "c74f7f2a-95ff-458c-9a03-8943860fcaef",
|
||
|
|
"sort_order": 0,
|
||
|
|
"rubric_text": "In mailbox.json (Sent folder), a sent email to sofia.ramirez@sunshineandsetauto.com must exist with the subject line 'Deal Gross Variance — Deal #10057 / Flores' and with body text \ncontaining the exact text 'Deal #10057', 'Flores', 'Stock #N16028', 'VIN ending '342FK', 'Approved gross: $7,500', 'Posted gross: $6,700', 'Variance: $800', and 'Source of difference', with Source of difference referencing accessory charge or desk recap error.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent emails found in mailbox.json.\"}\n \n # Primary target recipient per rubric\n target_recipients = [\n \"sofia.ramirez@sunshineandsetauto.com\",\n \"sofia.ramirez\",\n \"sofia\"\n ]\n \n best_match_email = None\n best_missing = None\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or email.get(\"to\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or email.get(\"cc\") or \"\").lower()\n bcc_addr = (email.get(\"bcc_addr\") or email.get(\"bcc\") or \"\").lower()\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n combined_addr = to_addr + \" \" + cc_addr + \" \" + bcc_addr\n combined_text = body + \" \" + subject\n \n # Check if sent to sofia.ramirez (primary requirement per rubric)\n recipient_match = any(r.lower() in combined_addr for r in target_recipients)\n if not recipient_match:\n continue\n \n missing = []\n \n # Check subject line contains deal gross variance and deal #10057 and Flores\n # Be lenient: accept various dash types (-, \\u2014, \\u2013) or no dash\n subject_has_deal_gross_variance = re.search(r'deal\\s*gross\\s*variance', subject)\n subject_has_10057 = re.search(r'10057', subject)\n subject_has_flores = 'flores' in subject\n if not subject_has_deal_gross_variance:\n missing.append(\"Subject must contain 'Deal Gross Variance'\")\n if not subject_has_10057:\n missing.append(\"Subject must contain 'Deal #10057'\")\n if not subject_has_flores:\n missing.append(\"Subject must contain 'Flores'\")\n \n # Deal #10057 in body - accept various formats\n if not re.search(r'10057', combined_text):\n missing.append(\"Body: Deal #10057\")\n \n # Flores in body\n if 'flores' not in combined_text:\n missing.append(\"Body: Flores\")\n \n # Stock #N16028 - accept N16028, N-16028, N 16028, etc.\n if not re.search(r'n[\\s\\-#]*16028', combined_text):\n missing.append(\"Body: Stock #N16028\")\n \n # VIN ending 342FK\n if not re.search(r'342fk', combined_text):\n missing.append(\"Body: VIN ending 342FK\")\n \n # Approved gross $7,500 - accept 7500, 7,500, 7500.00 etc.\n if not re.search(r'7[,.]?500', combined_text):\n missing.append(\"Body: Approved gross: $7,500\")\n \n # Posted gross $6,700 - accept 6700, 6,700, 6700.00 etc.\n if not re.search(r'6[,.]?700', combined_text):\n missing.append(\"Body: Posted gross: $6,700\")\n \n # Variance $800\n if not re.search(r'800', combined_text):\n missing.append(\"Body: Variance: $800\")\n \n # Source of difference - accessory charge or desk recap error\n accessory_keywords = [\"accessory\", \"accessories\", \"desk recap\", \"recap error\", \"recap\", \"add-on\", \"addon\", \"aftermarket\", \"source of difference\", \"discrepancy\", \"difference\", \"source\"]\n has_source = any(kw in combined_text for kw in accessory_keyw
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "25622b64-4ad0-41ae-9ed5-54e0a58e4fb2",
|
||
|
|
"sort_order": 1,
|
||
|
|
"rubric_text": "In the deal_log spreadsheet, row for Deal #10057 must contain text in the 'Notes' column (column AH) that states that the desk recap was in error and the deal_log total gross of $6,700 is correct (referencing Marcus Hale's resolution).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available.\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\")) + list(Path(workspace_path).glob(\"*.xlsm\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Excel files found in workspace.\"}\n\n deal_log_files = [f for f in xlsx_files if \"deal\" in f.name.lower() and \"log\" in f.name.lower()]\n if not deal_log_files:\n deal_log_files = xlsx_files\n\n # Keywords indicating the desk recap was in error (REQUIRED along with $6,700)\n keywords_error = [\"error\", \"incorrect\", \"in error\", \"was in error\", \"discrepancy\", \"wrong\", \"corrected\", \"correction\", \"mismatch\", \"adjusted\", \"override\", \"overridden\", \"resolved\", \"updated\", \"issue\", \"differs\", \"difference\"]\n # Keywords indicating $6,700 / deal_log is correct (REQUIRED)\n keywords_correct_amount = [\"6,700\", \"6700\", \"$6,700\", \"$6700\"]\n # Keywords referencing that the amount/deal_log is correct (supporting signal)\n keywords_correct_source = [\"correct\", \"deal_log\", \"deal log\", \"confirmed\", \"verified\", \"accurate\", \"actual\", \"true\", \"valid\", \"approved\", \"stands\", \"retained\"]\n # Keywords referencing Marcus Hale (supporting signal)\n keywords_marcus = [\"marcus\", \"hale\"]\n # Keywords referencing desk recap (supporting signal)\n keywords_recap = [\"recap\", \"desk\"]\n\n def _cell_str(v):\n return str(v).lower() if v is not None else \"\"\n\n def _find_col_index(header_row, candidates_substrings):\n \"\"\"Return 0-based index of first header cell whose lowercased text contains any candidate substring.\"\"\"\n for idx, cell in enumerate(header_row):\n val = _cell_str(cell.value).strip()\n if not val:\n continue\n for cand in candidates_substrings:\n if cand in val:\n return idx\n return None\n\n def _row_matches_deal(row, deal_col_idx, deal_number):\n if deal_col_idx is None or deal_col_idx >= len(row):\n return False\n raw = row[deal_col_idx].value\n if raw is None:\n return False\n try:\n return int(float(raw)) == deal_number\n except (TypeError, ValueError):\n return str(raw).strip() == str(deal_number)\n\n all_found_notes = []\n\n for fpath in deal_log_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n except Exception:\n continue\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n rows = list(ws.iter_rows())\n if not rows:\n continue\n\n header_row = rows[0]\n\n # Locate Notes column (prefer exact \"notes\" header)\n notes_col_idx = None\n for idx, cell in enumerate(header_row):\n val = _cell_str(cell.value).strip()\n if val == \"notes\" or val == \"note\" or val == \"comments\":\n notes_col_idx = idx\n break\n # Fallback: any header containing \"note\"\n if notes_col_idx is None:\n notes_col_idx = _find_col_index(header_row, [\"note\"])\n\n # Locate Deal # column\n deal_col_idx = None\n for idx, cell in enumerate(header_row):\n val = _cell_str(cell.value).strip()\n if val in (\"deal #\", \"deal#\", \"deal number\", \"deal no\", \"deal no.\", \"deal\"):\n deal_col_idx = idx\n break\n\n # Fallback column AH (index 33) if Notes header not found\n ah_col_idx = 33\n\n # If we can't find a deal column on this sheet, skip\n if deal_col_idx is None:\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "973ea6bf-95b0-4b14-853e-f4c57bc74ee5",
|
||
|
|
"sort_order": 2,
|
||
|
|
"rubric_text": "In the cash_exceptions_log spreadsheet, a new row must exist for the returned check with all required fields: original deposit date (03/12/2027) in column F, return date (03/25/2027) in column G, customer name Patricia Morris in column E, deal number 10091 in column D, check amount $2,300 in column J, and return reason code 'closed account' or 'account closed' in column L. MM/DD/YYYY formatting must be used for dates.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport re\nimport datetime\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available.\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\")) + list(Path(workspace_path).glob(\"*.xlsm\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Excel files found in workspace.\"}\n\n # Look for cash exception log files first, then fall back to all Excel files\n target_files = [f for f in xlsx_files if \"cash\" in f.name.lower() or \"exception\" in f.name.lower()]\n if not target_files:\n target_files = xlsx_files\n\n def date_to_strs(cell_value):\n \"\"\"Return multiple string representations for a date-like cell.\"\"\"\n results = []\n if cell_value is None:\n return [\"\"]\n if isinstance(cell_value, (datetime.datetime, datetime.date)):\n d = cell_value if isinstance(cell_value, datetime.date) else cell_value.date()\n results.append(d.strftime(\"%Y-%m-%d\"))\n results.append(d.strftime(\"%m/%d/%Y\"))\n results.append(d.strftime(\"%m/%d/%y\")) # MM/DD/YY\n results.append(f\"{d.month}/{d.day}/{d.year}\")\n results.append(f\"{d.month}/{d.day}/{str(d.year)[-2:]}\")\n results.append(d.strftime(\"%d-%b-%Y\").lower())\n results.append(d.strftime(\"%b %d\").lower())\n return results\n return [str(cell_value).lower().strip()]\n\n def check_date_match(cell_value, target_patterns):\n \"\"\"Check if a cell value matches any of the target date patterns.\"\"\"\n strs = date_to_strs(cell_value)\n for s in strs:\n for p in target_patterns:\n if p in s:\n return True\n # Also check plain string\n plain = str(cell_value).lower().strip() if cell_value is not None else \"\"\n for p in target_patterns:\n if p in plain:\n return True\n return False\n\n def cell_str(cell_value):\n if cell_value is None:\n return \"\"\n return str(cell_value).lower().strip()\n\n # Rubric says deposit date 03/12/2027 and return date 03/25/2027\n # Accept both MM/DD/YYYY and MM/DD/YY and other common formats\n deposit_date_patterns = [\"2027-03-12\", \"03/12/2027\", \"3/12/2027\", \"03/12/27\", \"3/12/27\", \"12-mar-2027\", \"12-mar\", \"mar 12\"]\n return_date_patterns = [\"2027-03-25\", \"03/25/2027\", \"3/25/2027\", \"03/25/27\", \"3/25/27\", \"25-mar-2027\", \"25-mar\", \"mar 25\"]\n\n best_match = None\n best_missing = None\n best_row_num = None\n best_found_details = None\n\n for fpath in target_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n except Exception:\n continue\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row_idx, row in enumerate(ws.iter_rows(), start=1):\n # Build row text for keyword searching\n row_strs = []\n for cell in row:\n row_strs.append(str(cell.value).lower() if cell.value is not None else \"\")\n row_text = \" \".join(row_strs)\n\n # Quick check: must have patricia/morris or 10091\n has_patricia = \"patricia\" in row_text and \"morris\" in row_text\n has_deal = \"10091\" in row_text\n\n if not (has_patricia or has_deal):\n continue\n\n missing = []\n found_details = []\n\n # Helper to get cell value by column index (1-indexed: A=1, D=4, etc.)\n def get_col(col_idx):\n \"\"\"Get cell value by 1-based column index from row tuple.\"\"\"\n if col_idx <= len(row):\n return row[col_
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "cee17bf8-c3ac-46f6-a2ac-b984b0c55823",
|
||
|
|
"sort_order": 3,
|
||
|
|
"rubric_text": "In mailbox.json (Sent folder) or slack_data.json, Marcus Hale and Elena Brooks must both be notified about the returned check for Patricia Morris / Deal 10091 / $2,300 (via email or Slack).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check mailbox or Slack.\"}\n \n marcus_notified = False\n elena_notified = False\n \n # Broad matching for email addresses / names\n marcus_ids = [\"marcus\", \"marcus.hale\", \"marcushale\", \"hale\"]\n elena_ids = [\"elena\", \"elena.brooks\", \"elenabrooks\", \"brooks\"]\n \n # Keywords that indicate the returned check context. Be loose: any two of these in the text is enough.\n returned_check_keywords = [\"returned\", \"return\", \"2,300\", \"2300\", \"patricia\", \"morris\", \"10091\", \"closed account\", \"nsf\", \"bounced\", \"check\", \"cheque\"]\n \n def text_has_returned_check_context(text):\n \"\"\"Return True if the text contains at least 2 of the returned-check keywords.\"\"\"\n matches = sum(1 for kw in returned_check_keywords if kw in text)\n return matches >= 2\n \n # ---- Check mailbox ----\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n bcc_addr = (email.get(\"bcc_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n combined_addr = to_addr + \" \" + cc_addr + \" \" + bcc_addr\n combined_text = body + \" \" + subject\n \n if not text_has_returned_check_context(combined_text):\n continue\n \n if any(m in combined_addr for m in marcus_ids):\n marcus_notified = True\n if any(e in combined_addr for e in elena_ids):\n elena_notified = True\n \n # ---- Check Slack ----\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n \n users = slack_data.get(\"users\", {})\n \n # Build sets of user IDs for Marcus and Elena\n marcus_user_ids = set()\n elena_user_ids = set()\n for uid, u in users.items():\n name_lower = (u.get(\"name\") or \"\").lower()\n real_name_lower = (u.get(\"real_name\") or \"\").lower()\n email_lower = (u.get(\"profile\", {}).get(\"email\") or \"\").lower()\n combined_user = name_lower + \" \" + real_name_lower + \" \" + email_lower\n if any(m in combined_user for m in [\"marcus\", \"hale\"]):\n marcus_user_ids.add(uid)\n if any(e in combined_user for e in [\"elena\", \"brooks\"]):\n elena_user_ids.add(uid)\n \n messages = slack_data.get(\"messages\", {})\n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n text = (msg.get(\"text\") or \"\").lower()\n if not text_has_returned_check_context(text):\n continue\n \n sender = msg.get(\"user\", \"\")\n \n # Check if Marcus is the recipient (mentioned in text, or channel is a DM with Marcus)\n if any(m in text for m in [\"marcus\", \"hale\", \"@marcus\"]):\n marcus_notified = True\n # If sent by someone TO a DM channel where Marcus is a member, or sender is marcus\n if sender in marcus_user_ids:\n # Mar
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "5df9f89c-6644-4a47-bf1f-4257422845a4",
|
||
|
|
"sort_order": 4,
|
||
|
|
"rubric_text": "In mailbox.json (Sent folder), a sent email to patricia2000@aol.com must exist notifying her about the returned check (referencing returned check, amount, or account issue).",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n returned_check_keywords = [\"returned\", \"return\", \"check\", \"payment\", \"2,300\", \"2300\", \"closed account\", \"account\", \"nsf\"]\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n \n if \"patricia2000@aol.com\" not in to_addr:\n continue\n \n combined_text = body + \" \" + subject\n has_returned_check = any(kw in combined_text for kw in returned_check_keywords)\n \n if has_returned_check:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Found sent email to patricia2000@aol.com notifying about the returned check.\"}\n else:\n return {\"pass\": False, \"score\": 0.3, \"feedback\": \"Found email to patricia2000@aol.com but it does not appear to mention the returned check or payment issue.\"}\n \n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to patricia2000@aol.com notifying about the returned check.\"}",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "c9d032ac-b740-4c98-9ff4-97622e136a16",
|
||
|
|
"sort_order": 5,
|
||
|
|
"rubric_text": "In the Q1 2027 sheet of the deal_log spreadsheet, Column AG (Title Status) must contain 'Held - Returned Check'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available.\"}\n\n def _cell_str(v):\n return str(v).strip().lower() if v is not None else \"\"\n\n # Look for any xlsx/xlsm files that could be the deal_log spreadsheet\n xlsx_files = list(Path(workspace_path).glob(\"**/*.xlsx\")) + list(Path(workspace_path).glob(\"**/*.xlsm\"))\n\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"No xlsx/xlsm files found in workspace: {workspace_path}\"}\n\n for fpath in xlsx_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n except Exception as e:\n continue\n\n # Look for a sheet that matches Q1 2027 (be lenient with naming)\n target_sheet = None\n for sheet_name in wb.sheetnames:\n sn_lower = sheet_name.strip().lower()\n if \"q1\" in sn_lower and \"2027\" in sn_lower:\n target_sheet = sheet_name\n break\n\n if target_sheet is None:\n # Also try sheets that just say \"q1 2027\" or similar\n for sheet_name in wb.sheetnames:\n sn_lower = sheet_name.strip().lower().replace(\" \", \"\")\n if \"q12027\" in sn_lower:\n target_sheet = sheet_name\n break\n\n if target_sheet is None:\n continue\n\n ws = wb[target_sheet]\n rows = list(ws.iter_rows())\n if not rows:\n continue\n\n # Column AG is the 33rd column (1-indexed), so 0-indexed it's 32\n ag_col_idx = 32 # 0-based index for column AG\n\n # Check every cell in column AG for the expected value\n found_values_ag = []\n for row in rows:\n if ag_col_idx < len(row):\n # Filter to only check deal #10091's row\n deal_num = row[0].value\n deal_str = str(deal_num).strip() if deal_num else \"\"\n if deal_str != \"10091\":\n continue\n cell_val = row[ag_col_idx].value\n cell_str = _cell_str(cell_val)\n found_values_ag.append(cell_val)\n\n # Be lenient: check if cell contains 'held' and 'returned check' (case insensitive)\n if \"held\" in cell_str and \"returned check\" in cell_str:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found 'Held - Returned Check' in Column AG (Title Status) of sheet '{target_sheet}' in file '{fpath.name}'. Cell value: '{cell_val}'.\"\n }\n\n # Secondary check: also look at header row to find a column labeled 'Title Status'\n # in case columns have been shifted\n header_row = rows[0] if rows else []\n title_status_col_idx = None\n for idx, cell in enumerate(header_row):\n val = _cell_str(cell.value)\n if \"title status\" in val or \"title_status\" in val or val == \"title\":\n title_status_col_idx = idx\n break\n\n if title_status_col_idx is not None and title_status_col_idx != ag_col_idx:\n for row in rows[1:]:\n if title_status_col_idx < len(row):\n # Filter to only check deal #10091's row\n deal_num = row[0].value\n deal_str = str(deal_num).strip() if deal_num else \"\"\n if deal_str != \"10091\":\n continue\n cell_val = row[title_status_col_idx].value\n cell_str = _cell_str(cell_val)\n if \"held\" in cell_str and \"returned check\" in cell_str:\n return {\n \"pass\": True,\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "b2797c29-020a-42ff-b53e-34ce79041991",
|
||
|
|
"sort_order": 6,
|
||
|
|
"rubric_text": "In slack_data.json, a message in #acct-deal-posting must exist referencing Deal #10099 and a gross variance between the F&I recap and deal log.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n \"\"\"\n Rubric: In slack_data.json, a message in #acct-deal-posting must exist\n referencing Deal #10099 and a gross variance between the F&I recap and deal log.\n \"\"\"\n\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check slack_data.json.\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path.\"}\n\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n\n channels = slack_data.get(\"channels\", {})\n messages = slack_data.get(\"messages\", {})\n\n # Find the #acct-deal-posting channel(s)\n target_channel_ids = [\n cid for cid, c in channels.items()\n if \"acct-deal-posting\" in (c.get(\"name\") or \"\").lower()\n ]\n\n if not target_channel_ids:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No #acct-deal-posting channel found in slack_data.json channels.\"}\n\n def text_has_deal_ref(text):\n \"\"\"Check if text references Deal #10099 (loosely).\"\"\"\n t = text.lower()\n return \"10099\" in t\n\n def text_has_gross_variance_ref(text):\n \"\"\"Check if text references a gross variance between F&I recap and deal log.\n \n We look for:\n 1. Direct mention of 'gross' combined with variance/discrepancy language\n 2. Mentions of F&I recap and deal log with variance language\n 3. Any combination of gross, variance, discrepancy, mismatch, difference keywords\n 4. Dollar amounts with variance/discrepancy context\n \"\"\"\n t = text.lower()\n\n # Variance / discrepancy keywords\n variance_words = [\n \"variance\", \"discrepanc\", \"difference\", \"mismatch\",\n \"does not match\", \"doesn't match\", \"don't match\", \"do not match\",\n \"doesn\\u2019t match\", \"don\\u2019t match\",\n \"off by\", \"incorrect\", \"error\", \"issue\", \"flag\",\n \"inconsisten\", \"disagree\", \"conflict\", \"differ\",\n \"not matching\", \"not align\", \"misalign\", \"gap\"\n ]\n has_variance = any(w in t for w in variance_words)\n\n # Gross keyword\n has_gross = \"gross\" in t\n\n # F&I recap keywords\n fi_recap_words = [\"f&i\", \"fi recap\", \"f and i\", \"f & i\", \"recap\"]\n has_fi_recap = any(w in t for w in fi_recap_words)\n\n # Deal log keywords\n deal_log_words = [\"deal log\", \"deallog\", \"deal sheet\", \"deal record\"]\n has_deal_log = any(w in t for w in deal_log_words)\n\n has_dollar = bool(re.search(r'\\$[\\d,]+\\.?\\d*', t))\n has_number = bool(re.search(r'\\d{3,}', t))\n\n # Best case: gross + variance\n if has_gross and has_variance:\n return True\n\n # F&I recap + deal log + any variance/discrepancy\n if has_fi_recap and has_deal_log and has_variance:\n return True\n\n # F&I recap + deal log + gross (implies comparison)\n if has_fi_recap and has_deal_log and has_gross:\n return True\n\n # Gross + F&I recap or deal log (implies variance context)\n if has_gross and (has_fi_recap or has_deal_log):\n return True\n\n # Variance with dollar amount and gross\n if has_variance and has_dollar and has_gross:\n return True\n\n # Variance with F&I or deal log reference and dollar amount\n if has_variance and (has_fi_recap or has_deal_log) and has_dollar:\n return True\n\n # Loose: gross + dollar amount + any context word\n context_words = [\"profit\", \"total\", \"front\", \"back\", \"posting\", \"review\"]\n has_context = any(w in t for w
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "d0da5a96-d087-48a7-9d36-6da54673deb9",
|
||
|
|
"sort_order": 7,
|
||
|
|
"rubric_text": "In mailbox.json (Sent folder), a reply email to Marcus Bell's 'New Deal Jacket' email must exist communicating that Deal #10099 has a gross variance requiring Marcus Hale review and cannot be posted yet. ",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check mailbox.\"}\n \n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found.\"}\n \n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n all_emails = data.get(\"emails\", [])\n \n # Find Marcus Bell's 'New Deal Jacket' email (the original inbound email)\n marcus_bell_email = None\n for email in all_emails:\n from_addr = (email.get(\"from_addr\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n folder = (email.get(\"folder\") or \"\").lower()\n if (\"bell\" in from_addr or \"marcus\" in from_addr) and (\"deal jacket\" in subject or \"new deal\" in subject):\n if folder != \"sent\":\n marcus_bell_email = email\n break\n \n # Find sent emails\n sent_emails = [e for e in all_emails if (e.get(\"folder\") or \"\").lower() == \"sent\"]\n \n if not sent_emails:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent emails found in mailbox.\"}\n \n best_match = None\n best_score = 0.0\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n in_reply_to = email.get(\"in_reply_to\") or \"\"\n combined_text = body + \" \" + subject\n all_recipients = to_addr + \" \" + cc_addr\n \n # Check if this email is related to Marcus Bell / the deal jacket topic\n is_to_bell = \"bell\" in all_recipients\n is_reply_to_bell = False\n if marcus_bell_email:\n original_message_id = marcus_bell_email.get(\"message_id\") or \"\"\n if in_reply_to and original_message_id and in_reply_to == original_message_id:\n is_reply_to_bell = True\n \n is_about_deal_jacket = \"deal jacket\" in combined_text or \"new deal\" in combined_text or \"10099\" in combined_text\n \n # Must be either a reply to Bell, sent to Bell, or at least mention Bell and the deal\n is_bell_related = is_reply_to_bell or is_to_bell or (\"bell\" in combined_text and is_about_deal_jacket)\n \n if not is_bell_related:\n continue\n \n # --- Rubric core: gross variance, Marcus Hale review, cannot be posted, Deal #10099 ---\n \n # Check for ANY mention of variance / discrepancy / mismatch (loosely)\n has_variance_language = any(kw in combined_text for kw in [\n \"variance\", \"discrepancy\", \"difference\", \"mismatch\",\n \"does not match\", \"doesn't match\", \"don't match\", \"do not match\",\n \"inconsisten\", \"differ\", \"off by\", \"gross\"\n ])\n \n # Check for specific $4 variance or the two amounts\n has_4_variance = any(kw in combined_text for kw in [\"$4\", \"4 dollar\", \"four dollar\", \"$4.00\", \"4.00 variance\", \"4 variance\"])\n has_4_variance = has_4_variance or bool(re.search(r'\\$4(?:\\b|\\.00)', combined_text))\n \n has_8220 = any(kw in combined_text for kw in [\"8,220\", \"8220\", \"$8,220\", \"$8220\"])\n has_8224 = any(kw in combined_text for kw in [\"8,224\", \"8224\", \"$8,224\", \"$8224\"])\n has_both_amounts = has_8220 and has_8224\n \n has_explicit_variance_reference = has_4_variance or has_both_amounts\n \n # Check cannot-post / hold language\n has_cannot_post = any(kw in combined_text for kw in [\n \"cannot post\", \"can't post\", \"cann
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "d483f22b-bd8b-4ae3-b44d-e2acd7d831b9",
|
||
|
|
"sort_order": 8,
|
||
|
|
"rubric_text": "In the Q1 2027 sheet of the deal_log spreadsheet, the figures for Deal #10099 must be unchanged: cell AC27 must contain exactly $2,161, cell AD27 must contain exactly $6,059 and cell AE27 must contain exactly $8,220.",
|
||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available.\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\")) + list(Path(workspace_path).glob(\"*.xlsm\"))\n if not xlsx_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No Excel files found in workspace.\"}\n\n deal_log_files = [f for f in xlsx_files if \"deal\" in f.name.lower() and \"log\" in f.name.lower()]\n if not deal_log_files:\n deal_log_files = xlsx_files\n\n def parse_dollar(val):\n \"\"\"Try to parse a cell value as a numeric dollar amount.\"\"\"\n if val is None:\n return None\n if isinstance(val, (int, float)):\n return round(float(val), 2)\n s = str(val).strip().replace(\"$\", \"\").replace(\",\", \"\").strip()\n try:\n return round(float(s), 2)\n except (ValueError, TypeError):\n return None\n\n for fpath in deal_log_files:\n try:\n wb = openpyxl.load_workbook(fpath, data_only=True)\n except Exception:\n continue\n\n # Look for a sheet that matches \"Q1 2027\" loosely\n target_sheet = None\n for sheet_name in wb.sheetnames:\n sn_lower = sheet_name.lower().replace(\" \", \"\")\n if \"q1\" in sn_lower and \"2027\" in sn_lower:\n target_sheet = sheet_name\n break\n\n if target_sheet is None:\n sheets_to_check = wb.sheetnames\n else:\n sheets_to_check = [target_sheet]\n\n for sheet_name in sheets_to_check:\n ws = wb[sheet_name]\n\n # The rubric says row 27. First verify deal 10099 is there.\n row_num = 27\n\n row_has_10099 = False\n for cell in ws[row_num]:\n if cell.value is not None and \"10099\" in str(cell.value):\n row_has_10099 = True\n break\n\n if not row_has_10099:\n # Try to find 10099 in other rows of this sheet\n found_row = None\n for row in ws.iter_rows():\n for cell in row:\n if cell.value is not None and \"10099\" in str(cell.value):\n found_row = cell.row\n break\n if found_row:\n break\n if found_row:\n row_num = found_row\n else:\n continue\n\n # AC = column 29, AD = column 30, AE = column 31\n val_ac = ws.cell(row=row_num, column=29).value\n val_ad = ws.cell(row=row_num, column=30).value\n val_ae = ws.cell(row=row_num, column=31).value\n\n num_ac = parse_dollar(val_ac)\n num_ad = parse_dollar(val_ad)\n num_ae = parse_dollar(val_ae)\n\n issues = []\n passes = []\n\n # Check AC = $2,161 (tolerance ±1.5)\n if num_ac is not None and abs(num_ac - 2161.0) < 1.5:\n passes.append(f\"AC{row_num} = ${num_ac} (expected $2,161) ✓\")\n else:\n issues.append(f\"AC{row_num} = {val_ac} (parsed: {num_ac}), expected $2,161\")\n\n # Check AD = $6,059 (tolerance ±1.5)\n if num_ad is not None and abs(num_ad - 6059.0) < 1.5:\n passes.append(f\"AD{row_num} = ${num_ad} (expected $6,059) ✓\")\n else:\n issues.append(f\"AD{row_num} = {val_ad} (parsed: {num_ad}), expected $6,059\")\n\n # Check AE = $8,220 (tolerance ±1.5)\n if num_ae is not None and abs(num_ae - 8220.0) < 1.5:\n passes.append(f\"AE{row_num} = ${num_ae} (expected $8,220) ✓\")\n else:\n issues.append(f\"AE{row_num} = {val_ae} (parsed: {num_ae}), expected $8,22
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776334609340",
|
||
|
|
"sort_order": 9,
|
||
|
|
"rubric_text": "No sent email replies or Slack messages must be sent regarding the number of entries in the cash exception log, from the already-read email with subject 'cash exception log' sent from elena.brooks@sunshineandsetauto.com regarding number of entries in the log. ",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n feedback_parts = []\n found_violation = False\n\n # Check external services (email)\n if external_services_path is not None:\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, 'r') as f:\n mailbox_data = json.load(f)\n emails = mailbox_data.get(\"emails\", [])\n # Look for sent emails that reference 'cash exception log' in subject or body\n for email in emails:\n folder = str(email.get(\"folder\", \"\")).lower()\n subject = str(email.get(\"subject\", \"\")).lower()\n body = str(email.get(\"body_text\", \"\") or email.get(\"body\", \"\")).lower()\n to_addr = str(email.get(\"to_addr\", \"\") or email.get(\"to\", \"\")).lower()\n if folder == \"sent\":\n # Check if the subject references 'cash exception log'\n if \"cash exception log\" in subject:\n found_violation = True\n feedback_parts.append(\n f\"VIOLATION: Found sent email with subject '{email.get('subject')}' \"\n f\"to '{email.get('to_addr', email.get('to', 'unknown'))}' referencing 'cash exception log'.\"\n )\n # Also check body referencing 'cash exception log' and 'entries' / 'number of entries'\n elif \"cash exception log\" in body and (\"entries\" in body or \"number of\" in body):\n found_violation = True\n feedback_parts.append(\n f\"VIOLATION: Found sent email to '{email.get('to_addr', email.get('to', 'unknown'))}' \"\n f\"with body referencing 'cash exception log' and entries.\"\n )\n if not found_violation:\n feedback_parts.append(\"No sent emails found referencing the 'cash exception log' topic. Good.\")\n else:\n feedback_parts.append(\"mailbox.json not found; cannot verify sent emails.\")\n else:\n feedback_parts.append(\"No external_services_path provided; cannot verify sent emails directly.\")\n\n # Check for Slack messages\n if external_services_path is not None:\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, 'r') as f:\n slack_data = json.load(f)\n # Correctly iterate over slack messages per the documented format:\n # messages is a dict keyed by channel ID -> list of message objects\n slack_messages_dict = slack_data.get(\"messages\", {})\n all_messages = []\n if isinstance(slack_messages_dict, dict):\n for channel_id, channel_msgs in slack_messages_dict.items():\n if isinstance(channel_msgs, list):\n all_messages.extend(channel_msgs)\n # Also check if there's a flat list (fallback for alternative formats)\n elif isinstance(slack_messages_dict, list):\n all_messages.extend(slack_messages_dict)\n\n slack_violation_found = False\n for msg in all_messages:\n msg_text = str(msg.get(\"text\", \"\") or msg.get(\"message\", \"\") or msg.get(\"body\", \"\")).lower()\n if \"cash exception log\" in msg_text and (\"entries\" in msg_text or \"number of\" in msg_text):\n found_violation = True\n slack_violation_found = True\n feedback_parts.append(\n f\"VIOLATION: Found Slack message referencing 'cash exception log' and entries: {msg.get('text', msg)}\"\n )\n if not slack_violation_found:\n feedback_parts.a
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776334854994",
|
||
|
|
"sort_order": 10,
|
||
|
|
"rubric_text": "No new sent emails or Slack messages must be sent regarding or in reply to the email with subject 'New vehicles total'.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n feedback_parts = []\n found_violation = False\n subject_pattern = re.compile(r'new\\s*vehicles?\\s*total', re.IGNORECASE)\n\n # Check sent emails in mailbox.json\n if external_services_path:\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n try:\n with open(mailbox_path, 'r') as f:\n mailbox_data = json.load(f)\n emails = mailbox_data.get(\"emails\", [])\n sent_emails = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n for email in sent_emails:\n subj = email.get(\"subject\", \"\")\n body = email.get(\"body_text\", \"\") or email.get(\"body\", \"\")\n # Check if the subject line or body references 'New vehicles total'\n if subject_pattern.search(str(subj)) or subject_pattern.search(str(body)):\n found_violation = True\n feedback_parts.append(\n f\"Found sent email (id={email.get('email_id')}) with subject '{subj}' \"\n f\"that references 'New vehicles total'.\"\n )\n if not found_violation:\n feedback_parts.append(f\"Checked {len(sent_emails)} sent email(s); none reference 'New vehicles total'.\")\n except Exception as e:\n feedback_parts.append(f\"Error reading mailbox.json: {e}\")\n else:\n feedback_parts.append(\"No mailbox.json found; assuming no sent emails.\")\n else:\n feedback_parts.append(\"No external_services_path provided; cannot check sent emails directly. Assuming pass for email check.\")\n\n # Check Slack messages\n if external_services_path:\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n try:\n with open(slack_path, 'r') as f:\n slack_data = json.load(f)\n messages = []\n if isinstance(slack_data, dict):\n # Handle standard slack_data.json format: messages is dict of channel_id -> list of messages\n msgs_section = slack_data.get(\"messages\", {})\n if isinstance(msgs_section, dict):\n for channel_id, channel_msgs in msgs_section.items():\n if isinstance(channel_msgs, list):\n messages.extend(channel_msgs)\n # Also check other top-level keys that might contain message lists\n for key, value in slack_data.items():\n if key == \"messages\":\n continue\n if isinstance(value, list):\n for item in value:\n if isinstance(item, dict) and (\"text\" in item or \"message\" in item or \"content\" in item):\n messages.append(item)\n elif isinstance(slack_data, list):\n messages = slack_data\n\n slack_violation_found = False\n for msg in messages:\n if isinstance(msg, dict):\n text = str(msg.get(\"text\", \"\") or msg.get(\"message\", \"\") or msg.get(\"body\", \"\") or msg.get(\"content\", \"\"))\n if subject_pattern.search(text):\n found_violation = True\n slack_violation_found = True\n feedback_parts.append(\n f\"Found Slack message referencing 'New vehicles total': {text[:200]}\"\n )\n if not slack_violation_found:\n feedback_parts.append(f\"Checked {l
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "rubric_1776778125459",
|
||
|
|
"sort_order": 11,
|
||
|
|
"rubric_text": "A sent email addressed to Marcus Hale (marcus.hale@sunshineandsetauto.com), or a Slack message tagging Marcus Hale, or a direct Slack message to Marcus Hale must exist, notifying him about Deal #10099 and a gross variance.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n findings = []\n found_email = False\n found_slack = False\n deal_mentioned = False\n variance_mentioned = False\n\n # Check sent emails\n if external_services_path:\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n try:\n with open(mailbox_path, \"r\") as f:\n mailbox = json.load(f)\n emails = mailbox.get(\"emails\", [])\n for email in emails:\n folder = (email.get(\"folder\") or \"\").lower()\n if folder != \"sent\":\n continue\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n combined = subject + \" \" + body\n all_recipients = to_addr + \" \" + cc_addr\n # Check if addressed to Marcus Hale\n if \"marcus.hale@sunshineandsetauto.com\" in all_recipients or \"marcus.hale\" in all_recipients or \"marcus hale\" in all_recipients:\n found_email = True\n findings.append(f\"Found sent email to: {email.get('to_addr')}, cc: {email.get('cc_addr', '')}, subject: {email.get('subject')}\")\n # Check for deal 10099\n if \"10099\" in combined:\n deal_mentioned = True\n # Check for gross variance mention - be very generous\n variance_keywords = [\"variance\", \"discrepancy\", \"mismatch\", \"difference\", \"gross\",\n \"does not match\", \"doesn't match\", \"doesn't match\",\n \"incorrect\", \"exception\", \"inconsisten\", \"error\",\n \"issue\", \"conflict\", \"off by\", \"posting\",\n \"not matching\", \"do not match\", \"don't match\",\n \"don't match\", \"deviation\", \"discrepanc\"]\n for kw in variance_keywords:\n if kw in combined:\n variance_mentioned = True\n break\n except Exception as e:\n findings.append(f\"Error reading mailbox: {e}\")\n else:\n findings.append(\"mailbox.json not found at external_services_path\")\n\n # Check Slack messages via slack_data.json\n if external_services_path:\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n try:\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n messages_dict = slack_data.get(\"messages\", {})\n for channel_id, msg_list in messages_dict.items():\n if not isinstance(msg_list, list):\n continue\n for msg in msg_list:\n text = (msg.get(\"text\") or \"\").lower()\n # Check for Marcus Hale mention\n if \"marcus\" in text and (\"hale\" in text or \"marcus.hale\" in text or \"@marcus\" in text):\n found_slack = True\n findings.append(f\"Found Slack message mentioning Marcus Hale in channel {channel_id}: {text[:100]}\")\n if \"10099\" in text:\n deal_mentioned = True\n variance_keywords = [\"variance\", \"discrepancy\", \"mismatch\", \"difference\", \"gross\",\n
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "new_p3_hold_notification",
|
||
|
|
"sort_order": 12,
|
||
|
|
"rubric_text": "In slack_data.json, a message in #acct-titles-billing (channel C003) must exist referencing Deal #10091 and a title hold or returned-check hold, indicating that pending title work must be suspended or halted. The message must include the deal number (10091) and reference the returned check, hold status, or need to suspend title/registration work.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if not external_services_path:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No external_services_path provided; cannot check slack_data.json.\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n\n messages = slack_data.get(\"messages\", {})\n c003_msgs = messages.get(\"C003\", [])\n\n if not c003_msgs:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No messages found in channel C003 (#acct-titles-billing).\"}\n\n # The seeded Priya Bennett message has ts 1806044400.000100 — skip it\n seeded_ts = \"1806044400.000100\"\n\n hold_keywords = [\n \"hold\", \"held\", \"returned check\", \"returned\", \"suspend\",\n \"halt\", \"stop\", \"block\", \"pending\", \"do not release\",\n \"do not submit\", \"title\"\n ]\n\n for msg in c003_msgs:\n ts = msg.get(\"ts\", \"\")\n if ts == seeded_ts:\n continue\n text = (msg.get(\"text\") or \"\").lower()\n\n has_deal = \"10091\" in text\n if not has_deal:\n continue\n\n keyword_count = sum(1 for kw in hold_keywords if kw in text)\n\n if keyword_count >= 2:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found message in #acct-titles-billing (C003) referencing Deal #10091 with hold/returned-check context ({keyword_count} keywords matched). Message: {text[:300]}\"\n }\n else:\n return {\n \"pass\": False,\n \"score\": 0.3,\n \"feedback\": f\"Found message in C003 mentioning 10091 but lacking sufficient hold/returned-check context (only {keyword_count} keyword(s) matched; need at least 2). Message: {text[:300]}\"\n }\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"No message in #acct-titles-billing (C003) references Deal #10091. Found {len(c003_msgs)} message(s) in channel.\"\n }\n",
|
||
|
|
"criterion_type": "expected_output"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "new_prior_quarter_check",
|
||
|
|
"sort_order": 13,
|
||
|
|
"rubric_text": "Agent correctly applied the standard $1,000 escalation threshold (not the reduced $500 prior-quarter threshold) because Deal #10091's delivery date (March 13, 2027) falls in Q1 2027 — the current quarter, not a prior quarter. Agent did NOT send an additional notification to Talia Nguyen.",
|
||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n \"\"\"\n Check that the agent did NOT incorrectly apply the prior-quarter exception.\n The prior-quarter exception (reduced $500 threshold + Talia Nguyen notification)\n does NOT apply because Deal #10091's delivery date (March 13, 2027) is in Q1 2027,\n which is the current quarter (not a prior quarter).\n \n PASS if: No notification was sent to Talia Nguyen about the returned check.\n FAIL if: Agent sent a notification to Talia Nguyen about the returned check for Deal #10091.\n \"\"\"\n if not external_services_path:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No external_services_path provided; cannot check for incorrect Talia Nguyen notification. Assuming correct behavior.\"}\n \n talia_notified = False\n evidence = []\n \n # Check mailbox for emails to Talia Nguyen about the returned check\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if mailbox_path.exists():\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n \n sent_emails = [e for e in data.get(\"emails\", []) if e.get(\"folder\", \"\").lower() == \"sent\"]\n \n talia_ids = [\"talia\", \"nguyen\", \"talia.nguyen\"]\n returned_check_keywords = [\"returned\", \"return\", \"check\", \"10091\", \"morris\", \"2,300\", \"2300\", \"closed account\", \"nsf\", \"bounced\", \"title-hold\", \"title hold\", \"priority\"]\n \n for email in sent_emails:\n to_addr = (email.get(\"to_addr\") or \"\").lower()\n cc_addr = (email.get(\"cc_addr\") or \"\").lower()\n bcc_addr = (email.get(\"bcc_addr\") or \"\").lower()\n body = (email.get(\"body_text\") or email.get(\"body\") or \"\").lower()\n subject = (email.get(\"subject\") or \"\").lower()\n combined_addr = to_addr + \" \" + cc_addr + \" \" + bcc_addr\n combined_text = body + \" \" + subject\n \n # Check if sent to Talia Nguyen\n is_to_talia = any(t in combined_addr for t in talia_ids)\n if not is_to_talia:\n continue\n \n # Check if it's about the returned check\n keyword_matches = sum(1 for kw in returned_check_keywords if kw in combined_text)\n if keyword_matches >= 2:\n talia_notified = True\n evidence.append(f\"Email to Talia Nguyen (to: {to_addr}) about returned check. Subject: {email.get('subject', '(none)')}\")\n \n # Check Slack for messages mentioning Talia Nguyen about the returned check\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if slack_path.exists():\n with open(slack_path, \"r\") as f:\n slack_data = json.load(f)\n \n messages = slack_data.get(\"messages\", {})\n users = slack_data.get(\"users\", {})\n \n # Find Talia Nguyen user IDs\n talia_user_ids = set()\n for uid, u in users.items():\n name_lower = (u.get(\"name\") or \"\").lower()\n real_name_lower = (u.get(\"real_name\") or \"\").lower()\n email_lower = (u.get(\"profile\", {}).get(\"email\") or \"\").lower()\n combined_user = name_lower + \" \" + real_name_lower + \" \" + email_lower\n if \"talia\" in combined_user or \"nguyen\" in combined_user:\n talia_user_ids.add(uid)\n \n returned_check_keywords = [\"returned\", \"return\", \"check\", \"10091\", \"morris\", \"2,300\", \"2300\", \"closed account\", \"nsf\", \"bounced\", \"title-hold\", \"title hold\", \"priority\"]\n \n for channel_id, msgs in messages.items():\n if not isinstance(msgs, list):\n continue\n for msg in msgs:\n text = (msg.get(\"text\") or \"\").lower()\n \n # Check if Talia is mentioned\n mentions_tal
|
||
|
|
"criterion_type": "incorrect_behavior"
|
||
|
|
}
|
||
|
|
]
|