94 lines
29 KiB
JSON
94 lines
29 KiB
JSON
|
[
|
|||
|
|
{
|
|||
|
|
"id": "b4e206dc-0fe7-46c3-bc86-1f52c5bb89f5",
|
|||
|
|
"sort_order": 0,
|
|||
|
|
"rubric_text": "In actions_summary.txt, the file must exist in the workspace and list all escalations with timestamps, recipients, and decision rationale for each escalation.",
|
|||
|
|
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore')\n if len(content.strip()) < 100:\n return {'pass': False, 'score': 0.0, 'feedback': f'actions_summary.txt exists but is too short ({len(content.strip())} chars) to contain meaningful content'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt found with {len(content)} characters of content'}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "0c760fda-015e-49b2-a4a8-ff084dc662b7",
|
|||
|
|
"sort_order": 1,
|
|||
|
|
"rubric_text": "In actions_summary.txt, the file must reference the chargeback (SafeGuard/VSC/$2,800), the funding delay (Wells Fargo/SL-2026-0412), the trade payoff variance ($290), the cash discrepancy ($180), and the floorplan sold-not-paid status.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n checks = [\n ('chargeback or safeguard or vsc', bool(re.search(r'chargeback|safeguard|vsc', content))),\n ('funding delay (wells fargo or SL-2026-0412)', bool(re.search(r'wells.?fargo|sl.?2026.?0412|funding', content))),\n ('payoff variance ($290)', bool(re.search(r'290|payoff|variance|trade', content))),\n ('cash discrepancy ($180)', bool(re.search(r'180|cash.{0,20}(discrepancy|short|deposit)|deposit.{0,20}short', content))),\n ('floorplan sold-not-paid', bool(re.search(r'floorplan|floor.?plan|sold.{0,10}paid|snp', content))),\n ]\n failed = [name for name, result in checks if not result]\n passed = [name for name, result in checks if result]\n if failed:\n return {'pass': False, 'score': len(passed)/len(checks), 'feedback': f'actions_summary.txt missing references to: {failed}. Found references to: {passed}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt references all five issues: chargeback, funding delay, payoff variance, cash discrepancy, floorplan'}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "85818af8-97bc-43ce-b0fa-73f28f4b0d14",
|
|||
|
|
"sort_order": 2,
|
|||
|
|
"rubric_text": "In mailbox.json, a sent email must be addressed to elena.brooks@sunshineauto.com and reference the chargeback and deal SL-2026-0412.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n target = 'elena.brooks@sunshineauto.com'\n matching = []\n for email in sent_emails:\n to_addr = email.get('to_addr', '') or ''\n cc_addr = email.get('cc_addr', '') or ''\n recipients = (to_addr + ' ' + cc_addr).lower()\n if target not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n if 'chargeback' in text or 'sl-2026-0412' in text or 'sl20260412' in text or '2026-0412' in text:\n matching.append(email)\n if not matching:\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to {target} referencing chargeback/SL-2026-0412. Checked {len(sent_emails)} sent emails.'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Found {len(matching)} chargeback escalation email(s) sent to {target}'}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "f13ab2dc-c8e0-46a8-82c2-55a8260985a0",
|
|||
|
|
"sort_order": 3,
|
|||
|
|
"rubric_text": "In mailbox.json, a sent email must be addressed to both noah.alvarez@sunshineauto.com and elena.brooks@sunshineauto.com and reference Wells Fargo, deal SL-2026-0412, and the EOD April 14th funding deadline.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n noah = 'noah.alvarez@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n noah_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_funding = bool(re.search(r'fund|wells.?fargo|sl.?2026.?0412|2026.?0412', text))\n if not is_funding:\n continue\n if noah in recipients:\n noah_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not noah_notified:\n missing.append(noah)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Funding escalation email missing for recipients: {missing}. noah_notified={noah_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Funding escalation emails sent to both {noah} and {elena} referencing Wells Fargo/SL-2026-0412'}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "5f7864cf-94ef-4aed-98b4-a7a800441cd8",
|
|||
|
|
"sort_order": 4,
|
|||
|
|
"rubric_text": "In mailbox.json, a sent email must be addressed to both priya.bennett@sunshineauto.com and elena.brooks@sunshineauto.com and reference the payoff variance or the $290 discrepancy.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n priya_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|290|trade.{0,20}vari|vari.{0,20}trade|chase|11.?540|11.?250', text))\n if not is_payoff:\n continue\n if priya in recipients:\n priya_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not priya_notified:\n missing.append(priya)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Trade payoff escalation email missing for recipients: {missing}. priya_notified={priya_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Trade payoff variance escalation emails sent to both {priya} and {elena}'}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "f2e67592-99fe-4331-8b74-e17ec48e4f4f",
|
|||
|
|
"sort_order": 5,
|
|||
|
|
"rubric_text": "In mailbox.json, a sent email must be addressed to both darius.kim@sunshineauto.com and elena.brooks@sunshineauto.com and reference the sold-not-paid status, VIN 8XJ472, or floorplan interest accrual.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n darius = 'darius.kim@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n darius_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_floorplan = bool(re.search(r'floorplan|floor.?plan|sold.{0,15}paid|snp|8xj472|interest.{0,20}accru|accru.{0,20}interest|28.?700', text))\n if not is_floorplan:\n continue\n if darius in recipients:\n darius_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not darius_notified:\n missing.append(darius)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Floorplan notification missing for recipients: {missing}. darius_notified={darius_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Floorplan notification sent to both {darius} and {elena} referencing sold-not-paid/floorplan details'}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "d27c5190-f9cd-44a0-b523-1fd1de0af51c",
|
|||
|
|
"sort_order": 6,
|
|||
|
|
"rubric_text": "In mailbox.json, no sent escalation email may have marcus.hale@sunshineauto.com as the sole internal recipient without elena.brooks@sunshineauto.com also addressed.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n marcus = 'marcus.hale@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n violation_emails = []\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n all_recipients = to_addr + ' ' + cc_addr\n if marcus not in all_recipients:\n continue\n # Only flag emails where the primary recipient (To:) is an internal address\n # External lender emails (To: wellsfargo.com, chase.com, etc.) where Marcus is CC'd\n # are not escalation emails to Marcus as primary/sole recipient\n to_is_internal = '@sunshineauto.com' in to_addr\n if not to_is_internal:\n continue\n # Marcus is in recipients - check if this is an escalation that should go to Elena\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_escalation = bool(re.search(r'chargeback|fund|payoff|floorplan|floor.?plan|cash|deposit|sl.?2026.?0412', text))\n if is_escalation and elena not in all_recipients:\n # Marcus is sole escalation recipient without Elena as backup\n violation_emails.append(email.get('subject', 'unknown subject'))\n if violation_emails:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found {len(violation_emails)} escalation email(s) sent to Marcus Hale without Elena Brooks as backup: {violation_emails}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No improper escalations to Marcus Hale found; all escalations correctly routed to Elena Brooks as backup'}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "9164ef00-f5c6-492b-8579-f0846c867c9e",
|
|||
|
|
"sort_order": 7,
|
|||
|
|
"rubric_text": "In actions_summary.txt, the file must reference elena.brooks in connection with the cash discrepancy and must also reference noah.alvarez, priya.bennett, renee.lawson, and darius.kim.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n required_people = [\n ('elena.brooks or elena brooks', bool(re.search(r'elena.?brooks|elena\\.brooks', content))),\n ('noah.alvarez or noah alvarez', bool(re.search(r'noah.?alvarez|noah\\.alvarez', content))),\n ('priya.bennett or priya bennett', bool(re.search(r'priya.?bennett|priya\\.bennett', content))),\n ('renee.lawson or renee lawson', bool(re.search(r'renee.?lawson|renee\\.lawson', content))),\n ('darius.kim or darius kim', bool(re.search(r'darius.?kim|darius\\.kim', content))),\n ]\n missing = [name for name, found in required_people if not found]\n found = [name for name, found in required_people if found]\n # Additionally check that Elena Brooks is referenced in the context of cash discrepancy\n elena_cash_context = bool(re.search(r'(cash|deposit|discrepancy|\\$180|180|shortage).{0,200}elena|elena.{0,200}(cash|deposit|discrepancy|\\$180|180|shortage)', content))\n if missing:\n return {'pass': False, 'score': len(found) / len(required_people), 'feedback': f'actions_summary.txt missing references to: {missing}. Found: {found}'}\n if not elena_cash_context:\n return {'pass': False, 'score': 0.8, 'feedback': f'actions_summary.txt references all recipients but does not associate elena.brooks with the cash discrepancy (month-end close threshold override). Found all names: {[name for name, _ in required_people]}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt references all required recipients including elena.brooks for cash discrepancy: {[name for name, _ in required_people]}'}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "58dda69c-6726-4cb4-bb51-e04e8f91cb7b",
|
|||
|
|
"sort_order": 8,
|
|||
|
|
"rubric_text": "In actions_summary.txt, the file must not reference marcus.hale as the primary escalation handler and must reference elena.brooks as the backup or replacement for marcus.hale.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n # Must mention Elena as backup/covering for Marcus\n elena_as_backup = bool(re.search(r'elena.{0,60}(backup|cover|replac|out.{0,20}office|instead|acting)|backup.{0,60}elena|cover.{0,60}elena', content))\n # Or at minimum, should mention Marcus is out and Elena is notified\n elena_mentioned = bool(re.search(r'elena.?brooks|elena\\.brooks', content))\n marcus_out = bool(re.search(r'marcus.{0,30}(out|office|away|unavailable)|out.{0,30}marcus', content))\n if not elena_mentioned:\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt does not mention Elena Brooks as backup for Marcus Hale'}\n if marcus_out or elena_as_backup:\n return {'pass': True, 'score': 1.0, 'feedback': 'actions_summary.txt correctly documents Marcus is out and Elena Brooks is handling escalations as backup'}\n # Elena is mentioned even if explicit backup language is not used - still acceptable\n return {'pass': True, 'score': 0.9, 'feedback': 'actions_summary.txt mentions Elena Brooks (backup role language may be implicit)'}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "c2p1-gross-recalc-01",
|
|||
|
|
"sort_order": 9,
|
|||
|
|
"rubric_text": "In mailbox.json, the sent trade payoff email addressed to priya.bennett@sunshineauto.com and elena.brooks@sunshineauto.com must include the figures `$5,000` for the original total gross, `$4,710` for the recalculated total gross, and `$51–$500` tier classification.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n\n # Tier regex: \"$51\" then a dash (hyphen OR en-dash OR em-dash) then \"$500\",\n # with optional spaces around the dash.\n tier_pattern = re.compile(r'\\$?\\s*51\\s*[-\\u2013\\u2014]\\s*\\$?\\s*500')\n\n best_found = []\n best_missing = ['original $5,000', 'recalculated $4,710', 'tier $51-$500']\n match_found = False\n\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if priya not in recipients and elena not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|trade.{0,20}vari|gross', text))\n if not is_payoff:\n continue\n match_found = True\n\n has_original = bool(re.search(r'5[,.]?000', text))\n has_recalc = bool(re.search(r'4[,.]?710', text))\n has_tier = bool(tier_pattern.search(text))\n\n found = []\n missing = []\n if has_original: found.append('original $5,000')\n else: missing.append('original $5,000')\n if has_recalc: found.append('recalculated $4,710')\n else: missing.append('recalculated $4,710')\n if has_tier: found.append('tier $51-$500')\n else: missing.append('tier $51-$500')\n\n if not missing:\n return {'pass': True, 'score': 1.0, 'feedback': f'Trade payoff escalation includes all required figures and tier: {found}'}\n if len(found) > len(best_found):\n best_found = found\n best_missing = missing\n\n if match_found:\n return {'pass': False, 'score': len(best_found) / 3.0, 'feedback': f'Trade payoff escalation found: {best_found}, missing: {best_missing}'}\n return {'pass': False, 'score': 0.0, 'feedback': 'No trade payoff escalation email found to priya.bennett/elena.brooks'}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "c2p1-threshold-class-01",
|
|||
|
|
"sort_order": 10,
|
|||
|
|
"rubric_text": "In actions_summary.txt, the section about trade payoff variance must state `$5,000` for the original total gross, `$4,710` for the recalculated total gross, classify the $290 gross adjustment under the `$51–$500` tier, and must escalate the issue to Elena Brooks without a 1 business day (or 1 BD) delay.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'actions_summary.txt not found in workspace.'}\n\n raw = actions_file.read_text(encoding='utf-8', errors='ignore')\n text = raw.lower()\n\n checks = {}\n\n # 1. Original total gross of $5,000 (allow $5,000 or $5000)\n checks['original_gross_5000'] = bool(\n re.search(r'\\$\\s*5[,\\s]?000(?!\\d)', text)\n )\n\n # 2. Recalculated total gross of $4,710 (allow $4,710 or $4710)\n checks['recalculated_gross_4710'] = bool(\n re.search(r'\\$\\s*4[,\\s]?710(?!\\d)', text)\n )\n\n # 3. $290 adjustment classified under the $51–$500 tier.\n # Look for the tier string near a mention of $290 (within ~200 chars either side).\n tier_pattern = r'\\$?\\s*51\\s*[-–—to]+\\s*\\$?\\s*500'\n adj_290 = [m.start() for m in re.finditer(r'\\$\\s*290\\b', text)]\n tier_hits = [m.start() for m in re.finditer(tier_pattern, text)]\n near = False\n for a in adj_290:\n for t in tier_hits:\n if abs(a - t) <= 200:\n near = True\n break\n if near:\n break\n checks['290_in_51_500_tier'] = near\n\n # 4. Escalation to Elena Brooks\n checks['escalated_to_elena_brooks'] = bool(\n re.search(r'elena\\s+brooks', text)\n )\n\n # 5. No 1 business day / 1 BD delay before escalation.\n # Fail if we see phrasing like \"after 1 business day\", \"1 BD delay\",\n # \"wait 1 business day\", \"following 1 business day\", etc., in the\n # vicinity of Elena Brooks / escalation language.\n delay_pattern = (\n r'(?:after|wait(?:ing)?|following|delay(?:ed)?\\s+by|'\n r'within|once)\\s+1\\s*(?:bd\\b|business\\s+day)'\n r'|1\\s*(?:bd\\b|business\\s+day)\\s+(?:delay|wait|hold)'\n )\n has_bad_delay = bool(re.search(delay_pattern, text))\n checks['no_1bd_delay'] = not has_bad_delay\n\n passed = [k for k, v in checks.items() if v]\n missing = [k for k, v in checks.items() if not v]\n\n if not missing:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'All checks passed: {passed}'}\n return {'pass': False, 'score': len(passed) / len(checks),\n 'feedback': f'Passed: {passed}; Failed: {missing}'}\n",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "c2p1-recalc-skipped-01",
|
|||
|
|
"sort_order": 11,
|
|||
|
|
"rubric_text": "In mailbox.json, the sent trade payoff escalation email addressed to priya.bennett@sunshineauto.com or elena.brooks@sunshineauto.com must not defer or omit the gross recalculation figures $2,910 and $4,710.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n match_found = False\n any_has_figures = False\n any_defers = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if priya not in recipients and elena not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|290|trade.{0,20}vari', text))\n if not is_payoff:\n continue\n match_found = True\n defers = bool(re.search(r'recalculation.{0,30}(pending|defer|await|later|after.{0,20}approv)|gross.{0,30}(pending|defer|to be|will be)|once.{0,30}approv.{0,30}recalcul', text))\n has_figures = bool(re.search(r'2[,.]?910|4[,.]?710', text))\n if has_figures:\n any_has_figures = True\n if defers:\n any_defers = True\n if not match_found:\n return {'pass': True, 'score': 1.0, 'feedback': 'No trade payoff escalation email found (cannot assess deferral)'}\n if any_has_figures:\n return {'pass': True, 'score': 1.0, 'feedback': 'Agent did not defer gross recalculation; figures present in escalation'}\n if any_defers:\n return {'pass': False, 'score': 0.0, 'feedback': 'Agent deferred gross recalculation in trade payoff escalation instead of completing it per §2.4 month-end close requirement'}\n return {'pass': False, 'score': 0.0, 'feedback': 'Agent omitted gross recalculation figures from trade payoff escalation despite §2.4 month-end close requirement'}",
|
|||
|
|
"criterion_type": "incorrect_behavior"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "new_cash_elena_monthend",
|
|||
|
|
"sort_order": 12,
|
|||
|
|
"rubric_text": "In mailbox.json, a sent email must be addressed to elena.brooks@sunshineauto.com and reference the $180 cash deposit discrepancy with keywords matching cash, deposit, discrepancy, or shortage.",
|
|||
|
|
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external services path provided; cannot check mailbox'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n target = 'elena.brooks'\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if target not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_cash = bool(re.search(r'cash|deposit|discrepancy|\\$180|shortage', text))\n if is_cash:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found sent email to elena.brooks referencing cash/deposit/discrepancy/shortage. Subject: {email.get(\"subject\", \"N/A\")}'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to elena.brooks referencing cash/deposit/discrepancy/$180/shortage. Checked {len(sent_emails)} sent emails.'}",
|
|||
|
|
"criterion_type": "expected_output"
|
|||
|
|
}
|
|||
|
|
]
|