Add Handbook.md benchmark tasks

This commit is contained in:
DerekSurge
2026-06-24 12:44:34 -07:00
commit 72f9a65917
1800 changed files with 323589 additions and 0 deletions
@@ -0,0 +1,93 @@
[
{
"id": "b4e206dc-0fe7-46c3-bc86-1f52c5bb89f5",
"sort_order": 0,
"rubric_text": "In actions_summary.txt, the file must exist in the workspace and list all escalations with timestamps, recipients, and decision rationale for each escalation.",
"verifier_code": "from pathlib import Path\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore')\n if len(content.strip()) < 100:\n return {'pass': False, 'score': 0.0, 'feedback': f'actions_summary.txt exists but is too short ({len(content.strip())} chars) to contain meaningful content'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt found with {len(content)} characters of content'}",
"criterion_type": "expected_output"
},
{
"id": "0c760fda-015e-49b2-a4a8-ff084dc662b7",
"sort_order": 1,
"rubric_text": "In actions_summary.txt, the file must reference the chargeback (SafeGuard/VSC/$2,800), the funding delay (Wells Fargo/SL-2026-0412), the trade payoff variance ($290), the cash discrepancy ($180), and the floorplan sold-not-paid status.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n checks = [\n ('chargeback or safeguard or vsc', bool(re.search(r'chargeback|safeguard|vsc', content))),\n ('funding delay (wells fargo or SL-2026-0412)', bool(re.search(r'wells.?fargo|sl.?2026.?0412|funding', content))),\n ('payoff variance ($290)', bool(re.search(r'290|payoff|variance|trade', content))),\n ('cash discrepancy ($180)', bool(re.search(r'180|cash.{0,20}(discrepancy|short|deposit)|deposit.{0,20}short', content))),\n ('floorplan sold-not-paid', bool(re.search(r'floorplan|floor.?plan|sold.{0,10}paid|snp', content))),\n ]\n failed = [name for name, result in checks if not result]\n passed = [name for name, result in checks if result]\n if failed:\n return {'pass': False, 'score': len(passed)/len(checks), 'feedback': f'actions_summary.txt missing references to: {failed}. Found references to: {passed}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt references all five issues: chargeback, funding delay, payoff variance, cash discrepancy, floorplan'}",
"criterion_type": "expected_output"
},
{
"id": "85818af8-97bc-43ce-b0fa-73f28f4b0d14",
"sort_order": 2,
"rubric_text": "In mailbox.json, a sent email must be addressed to elena.brooks@sunshineauto.com and reference the chargeback and deal SL-2026-0412.",
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n target = 'elena.brooks@sunshineauto.com'\n matching = []\n for email in sent_emails:\n to_addr = email.get('to_addr', '') or ''\n cc_addr = email.get('cc_addr', '') or ''\n recipients = (to_addr + ' ' + cc_addr).lower()\n if target not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n if 'chargeback' in text or 'sl-2026-0412' in text or 'sl20260412' in text or '2026-0412' in text:\n matching.append(email)\n if not matching:\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to {target} referencing chargeback/SL-2026-0412. Checked {len(sent_emails)} sent emails.'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Found {len(matching)} chargeback escalation email(s) sent to {target}'}",
"criterion_type": "expected_output"
},
{
"id": "f13ab2dc-c8e0-46a8-82c2-55a8260985a0",
"sort_order": 3,
"rubric_text": "In mailbox.json, a sent email must be addressed to both noah.alvarez@sunshineauto.com and elena.brooks@sunshineauto.com and reference Wells Fargo, deal SL-2026-0412, and the EOD April 14th funding deadline.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n noah = 'noah.alvarez@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n noah_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_funding = bool(re.search(r'fund|wells.?fargo|sl.?2026.?0412|2026.?0412', text))\n if not is_funding:\n continue\n if noah in recipients:\n noah_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not noah_notified:\n missing.append(noah)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Funding escalation email missing for recipients: {missing}. noah_notified={noah_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Funding escalation emails sent to both {noah} and {elena} referencing Wells Fargo/SL-2026-0412'}",
"criterion_type": "expected_output"
},
{
"id": "5f7864cf-94ef-4aed-98b4-a7a800441cd8",
"sort_order": 4,
"rubric_text": "In mailbox.json, a sent email must be addressed to both priya.bennett@sunshineauto.com and elena.brooks@sunshineauto.com and reference the payoff variance or the $290 discrepancy.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n priya_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|290|trade.{0,20}vari|vari.{0,20}trade|chase|11.?540|11.?250', text))\n if not is_payoff:\n continue\n if priya in recipients:\n priya_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not priya_notified:\n missing.append(priya)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Trade payoff escalation email missing for recipients: {missing}. priya_notified={priya_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Trade payoff variance escalation emails sent to both {priya} and {elena}'}",
"criterion_type": "expected_output"
},
{
"id": "f2e67592-99fe-4331-8b74-e17ec48e4f4f",
"sort_order": 5,
"rubric_text": "In mailbox.json, a sent email must be addressed to both darius.kim@sunshineauto.com and elena.brooks@sunshineauto.com and reference the sold-not-paid status, VIN 8XJ472, or floorplan interest accrual.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n darius = 'darius.kim@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n darius_notified = False\n elena_notified = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_floorplan = bool(re.search(r'floorplan|floor.?plan|sold.{0,15}paid|snp|8xj472|interest.{0,20}accru|accru.{0,20}interest|28.?700', text))\n if not is_floorplan:\n continue\n if darius in recipients:\n darius_notified = True\n if elena in recipients:\n elena_notified = True\n missing = []\n if not darius_notified:\n missing.append(darius)\n if not elena_notified:\n missing.append(elena)\n if missing:\n return {'pass': False, 'score': (2 - len(missing)) / 2, 'feedback': f'Floorplan notification missing for recipients: {missing}. darius_notified={darius_notified}, elena_notified={elena_notified}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'Floorplan notification sent to both {darius} and {elena} referencing sold-not-paid/floorplan details'}",
"criterion_type": "expected_output"
},
{
"id": "d27c5190-f9cd-44a0-b523-1fd1de0af51c",
"sort_order": 6,
"rubric_text": "In mailbox.json, no sent escalation email may have marcus.hale@sunshineauto.com as the sole internal recipient without elena.brooks@sunshineauto.com also addressed.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n marcus = 'marcus.hale@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n violation_emails = []\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n all_recipients = to_addr + ' ' + cc_addr\n if marcus not in all_recipients:\n continue\n # Only flag emails where the primary recipient (To:) is an internal address\n # External lender emails (To: wellsfargo.com, chase.com, etc.) where Marcus is CC'd\n # are not escalation emails to Marcus as primary/sole recipient\n to_is_internal = '@sunshineauto.com' in to_addr\n if not to_is_internal:\n continue\n # Marcus is in recipients - check if this is an escalation that should go to Elena\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_escalation = bool(re.search(r'chargeback|fund|payoff|floorplan|floor.?plan|cash|deposit|sl.?2026.?0412', text))\n if is_escalation and elena not in all_recipients:\n # Marcus is sole escalation recipient without Elena as backup\n violation_emails.append(email.get('subject', 'unknown subject'))\n if violation_emails:\n return {'pass': False, 'score': 0.0, 'feedback': f'Found {len(violation_emails)} escalation email(s) sent to Marcus Hale without Elena Brooks as backup: {violation_emails}'}\n return {'pass': True, 'score': 1.0, 'feedback': 'No improper escalations to Marcus Hale found; all escalations correctly routed to Elena Brooks as backup'}",
"criterion_type": "incorrect_behavior"
},
{
"id": "9164ef00-f5c6-492b-8579-f0846c867c9e",
"sort_order": 7,
"rubric_text": "In actions_summary.txt, the file must reference elena.brooks in connection with the cash discrepancy and must also reference noah.alvarez, priya.bennett, renee.lawson, and darius.kim.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n required_people = [\n ('elena.brooks or elena brooks', bool(re.search(r'elena.?brooks|elena\\.brooks', content))),\n ('noah.alvarez or noah alvarez', bool(re.search(r'noah.?alvarez|noah\\.alvarez', content))),\n ('priya.bennett or priya bennett', bool(re.search(r'priya.?bennett|priya\\.bennett', content))),\n ('renee.lawson or renee lawson', bool(re.search(r'renee.?lawson|renee\\.lawson', content))),\n ('darius.kim or darius kim', bool(re.search(r'darius.?kim|darius\\.kim', content))),\n ]\n missing = [name for name, found in required_people if not found]\n found = [name for name, found in required_people if found]\n # Additionally check that Elena Brooks is referenced in the context of cash discrepancy\n elena_cash_context = bool(re.search(r'(cash|deposit|discrepancy|\\$180|180|shortage).{0,200}elena|elena.{0,200}(cash|deposit|discrepancy|\\$180|180|shortage)', content))\n if missing:\n return {'pass': False, 'score': len(found) / len(required_people), 'feedback': f'actions_summary.txt missing references to: {missing}. Found: {found}'}\n if not elena_cash_context:\n return {'pass': False, 'score': 0.8, 'feedback': f'actions_summary.txt references all recipients but does not associate elena.brooks with the cash discrepancy (month-end close threshold override). Found all names: {[name for name, _ in required_people]}'}\n return {'pass': True, 'score': 1.0, 'feedback': f'actions_summary.txt references all required recipients including elena.brooks for cash discrepancy: {[name for name, _ in required_people]}'}",
"criterion_type": "expected_output"
},
{
"id": "58dda69c-6726-4cb4-bb51-e04e8f91cb7b",
"sort_order": 8,
"rubric_text": "In actions_summary.txt, the file must not reference marcus.hale as the primary escalation handler and must reference elena.brooks as the backup or replacement for marcus.hale.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt not found in workspace'}\n content = actions_file.read_text(encoding='utf-8', errors='ignore').lower()\n # Must mention Elena as backup/covering for Marcus\n elena_as_backup = bool(re.search(r'elena.{0,60}(backup|cover|replac|out.{0,20}office|instead|acting)|backup.{0,60}elena|cover.{0,60}elena', content))\n # Or at minimum, should mention Marcus is out and Elena is notified\n elena_mentioned = bool(re.search(r'elena.?brooks|elena\\.brooks', content))\n marcus_out = bool(re.search(r'marcus.{0,30}(out|office|away|unavailable)|out.{0,30}marcus', content))\n if not elena_mentioned:\n return {'pass': False, 'score': 0.0, 'feedback': 'actions_summary.txt does not mention Elena Brooks as backup for Marcus Hale'}\n if marcus_out or elena_as_backup:\n return {'pass': True, 'score': 1.0, 'feedback': 'actions_summary.txt correctly documents Marcus is out and Elena Brooks is handling escalations as backup'}\n # Elena is mentioned even if explicit backup language is not used - still acceptable\n return {'pass': True, 'score': 0.9, 'feedback': 'actions_summary.txt mentions Elena Brooks (backup role language may be implicit)'}",
"criterion_type": "incorrect_behavior"
},
{
"id": "c2p1-gross-recalc-01",
"sort_order": 9,
"rubric_text": "In mailbox.json, the sent trade payoff email addressed to priya.bennett@sunshineauto.com and elena.brooks@sunshineauto.com must include the figures `$5,000` for the original total gross, `$4,710` for the recalculated total gross, and `$51$500` tier classification.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping email check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping email check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n\n # Tier regex: \"$51\" then a dash (hyphen OR en-dash OR em-dash) then \"$500\",\n # with optional spaces around the dash.\n tier_pattern = re.compile(r'\\$?\\s*51\\s*[-\\u2013\\u2014]\\s*\\$?\\s*500')\n\n best_found = []\n best_missing = ['original $5,000', 'recalculated $4,710', 'tier $51-$500']\n match_found = False\n\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if priya not in recipients and elena not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|trade.{0,20}vari|gross', text))\n if not is_payoff:\n continue\n match_found = True\n\n has_original = bool(re.search(r'5[,.]?000', text))\n has_recalc = bool(re.search(r'4[,.]?710', text))\n has_tier = bool(tier_pattern.search(text))\n\n found = []\n missing = []\n if has_original: found.append('original $5,000')\n else: missing.append('original $5,000')\n if has_recalc: found.append('recalculated $4,710')\n else: missing.append('recalculated $4,710')\n if has_tier: found.append('tier $51-$500')\n else: missing.append('tier $51-$500')\n\n if not missing:\n return {'pass': True, 'score': 1.0, 'feedback': f'Trade payoff escalation includes all required figures and tier: {found}'}\n if len(found) > len(best_found):\n best_found = found\n best_missing = missing\n\n if match_found:\n return {'pass': False, 'score': len(best_found) / 3.0, 'feedback': f'Trade payoff escalation found: {best_found}, missing: {best_missing}'}\n return {'pass': False, 'score': 0.0, 'feedback': 'No trade payoff escalation email found to priya.bennett/elena.brooks'}\n",
"criterion_type": "expected_output"
},
{
"id": "c2p1-threshold-class-01",
"sort_order": 10,
"rubric_text": "In actions_summary.txt, the section about trade payoff variance must state `$5,000` for the original total gross, `$4,710` for the recalculated total gross, classify the $290 gross adjustment under the `$51$500` tier, and must escalate the issue to Elena Brooks without a 1 business day (or 1 BD) delay.",
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n actions_file = Path(workspace_path) / 'actions_summary.txt'\n if not actions_file.exists():\n return {'pass': False, 'score': 0.0,\n 'feedback': 'actions_summary.txt not found in workspace.'}\n\n raw = actions_file.read_text(encoding='utf-8', errors='ignore')\n text = raw.lower()\n\n checks = {}\n\n # 1. Original total gross of $5,000 (allow $5,000 or $5000)\n checks['original_gross_5000'] = bool(\n re.search(r'\\$\\s*5[,\\s]?000(?!\\d)', text)\n )\n\n # 2. Recalculated total gross of $4,710 (allow $4,710 or $4710)\n checks['recalculated_gross_4710'] = bool(\n re.search(r'\\$\\s*4[,\\s]?710(?!\\d)', text)\n )\n\n # 3. $290 adjustment classified under the $51$500 tier.\n # Look for the tier string near a mention of $290 (within ~200 chars either side).\n tier_pattern = r'\\$?\\s*51\\s*[-–—to]+\\s*\\$?\\s*500'\n adj_290 = [m.start() for m in re.finditer(r'\\$\\s*290\\b', text)]\n tier_hits = [m.start() for m in re.finditer(tier_pattern, text)]\n near = False\n for a in adj_290:\n for t in tier_hits:\n if abs(a - t) <= 200:\n near = True\n break\n if near:\n break\n checks['290_in_51_500_tier'] = near\n\n # 4. Escalation to Elena Brooks\n checks['escalated_to_elena_brooks'] = bool(\n re.search(r'elena\\s+brooks', text)\n )\n\n # 5. No 1 business day / 1 BD delay before escalation.\n # Fail if we see phrasing like \"after 1 business day\", \"1 BD delay\",\n # \"wait 1 business day\", \"following 1 business day\", etc., in the\n # vicinity of Elena Brooks / escalation language.\n delay_pattern = (\n r'(?:after|wait(?:ing)?|following|delay(?:ed)?\\s+by|'\n r'within|once)\\s+1\\s*(?:bd\\b|business\\s+day)'\n r'|1\\s*(?:bd\\b|business\\s+day)\\s+(?:delay|wait|hold)'\n )\n has_bad_delay = bool(re.search(delay_pattern, text))\n checks['no_1bd_delay'] = not has_bad_delay\n\n passed = [k for k, v in checks.items() if v]\n missing = [k for k, v in checks.items() if not v]\n\n if not missing:\n return {'pass': True, 'score': 1.0,\n 'feedback': f'All checks passed: {passed}'}\n return {'pass': False, 'score': len(passed) / len(checks),\n 'feedback': f'Passed: {passed}; Failed: {missing}'}\n",
"criterion_type": "expected_output"
},
{
"id": "c2p1-recalc-skipped-01",
"sort_order": 11,
"rubric_text": "In mailbox.json, the sent trade payoff escalation email addressed to priya.bennett@sunshineauto.com or elena.brooks@sunshineauto.com must not defer or omit the gross recalculation figures $2,910 and $4,710.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': True, 'score': 1.0, 'feedback': 'No external services path provided; skipping check'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': True, 'score': 1.0, 'feedback': 'mailbox.json not found; skipping check'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n priya = 'priya.bennett@sunshineauto.com'\n elena = 'elena.brooks@sunshineauto.com'\n match_found = False\n any_has_figures = False\n any_defers = False\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if priya not in recipients and elena not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_payoff = bool(re.search(r'payoff|290|trade.{0,20}vari', text))\n if not is_payoff:\n continue\n match_found = True\n defers = bool(re.search(r'recalculation.{0,30}(pending|defer|await|later|after.{0,20}approv)|gross.{0,30}(pending|defer|to be|will be)|once.{0,30}approv.{0,30}recalcul', text))\n has_figures = bool(re.search(r'2[,.]?910|4[,.]?710', text))\n if has_figures:\n any_has_figures = True\n if defers:\n any_defers = True\n if not match_found:\n return {'pass': True, 'score': 1.0, 'feedback': 'No trade payoff escalation email found (cannot assess deferral)'}\n if any_has_figures:\n return {'pass': True, 'score': 1.0, 'feedback': 'Agent did not defer gross recalculation; figures present in escalation'}\n if any_defers:\n return {'pass': False, 'score': 0.0, 'feedback': 'Agent deferred gross recalculation in trade payoff escalation instead of completing it per §2.4 month-end close requirement'}\n return {'pass': False, 'score': 0.0, 'feedback': 'Agent omitted gross recalculation figures from trade payoff escalation despite §2.4 month-end close requirement'}",
"criterion_type": "incorrect_behavior"
},
{
"id": "new_cash_elena_monthend",
"sort_order": 12,
"rubric_text": "In mailbox.json, a sent email must be addressed to elena.brooks@sunshineauto.com and reference the $180 cash deposit discrepancy with keywords matching cash, deposit, discrepancy, or shortage.",
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {'pass': False, 'score': 0.0, 'feedback': 'No external services path provided; cannot check mailbox'}\n mailbox_path = Path(external_services_path) / 'mailbox.json'\n if not mailbox_path.exists():\n return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found'}\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n sent_emails = [e for e in data.get('emails', []) if e.get('folder', '').lower() == 'sent']\n target = 'elena.brooks'\n for email in sent_emails:\n to_addr = (email.get('to_addr', '') or '').lower()\n cc_addr = (email.get('cc_addr', '') or '').lower()\n recipients = to_addr + ' ' + cc_addr\n if target not in recipients:\n continue\n body = (email.get('body_text', '') or '').lower()\n subject = (email.get('subject', '') or '').lower()\n text = body + ' ' + subject\n is_cash = bool(re.search(r'cash|deposit|discrepancy|\\$180|shortage', text))\n if is_cash:\n return {'pass': True, 'score': 1.0, 'feedback': f'Found sent email to elena.brooks referencing cash/deposit/discrepancy/shortage. Subject: {email.get(\"subject\", \"N/A\")}'}\n return {'pass': False, 'score': 0.0, 'feedback': f'No sent email found to elena.brooks referencing cash/deposit/discrepancy/$180/shortage. Checked {len(sent_emails)} sent emails.'}",
"criterion_type": "expected_output"
}
]
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Run bundled SOP Python verifiers inside a Harbor task container."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Any
WORKDIR = Path("/workdir")
DATA_DIR = Path("/data")
INITIAL_DATA_DIR = Path("/initial_data")
TESTS_DIR = Path("/tests")
VERIFIER_DIR = Path("/logs/verifier")
SERVICE_COMPAT_FILES: dict[str, tuple[str, tuple[str, ...]]] = {
"slack": ("slack.json", ("slack.json", "slack_data.json")),
"google_mail": ("inbox.json", ("inbox.json", "mailbox.json")),
"google_calendar": ("calendar_data.json", ("calendar_data.json", "calendar.json")),
"jira": ("jira_state.json", ("jira_state.json", "jira_data.json")),
"shopify": ("shopify_data.json", ("shopify_data.json",)),
}
def _state_path(service: str, seed_name: str) -> Path | None:
candidates = [
DATA_DIR / service / "final.json",
DATA_DIR / service / seed_name,
INITIAL_DATA_DIR / service / seed_name,
]
return next((p for p in candidates if p.is_file()), None)
def _build_compat_external_services(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for service, (seed_name, compat_names) in SERVICE_COMPAT_FILES.items():
src = _state_path(service, seed_name)
if src is not None:
for compat_name in compat_names:
shutil.copy2(src, dest / compat_name)
def _coerce_result(raw: Any) -> dict[str, Any]:
if isinstance(raw, dict):
passed = bool(raw.get("pass", raw.get("passed", False)))
score = raw.get("score", 1.0 if passed else 0.0)
try:
score = float(score)
except (TypeError, ValueError):
score = 1.0 if passed else 0.0
return {
"pass": passed,
"score": max(0.0, min(1.0, score)),
"feedback": str(raw.get("feedback", "")),
}
passed = bool(raw)
return {"pass": passed, "score": 1.0 if passed else 0.0, "feedback": str(raw)}
def _run_one(rubric: dict[str, Any], external_services_path: Path) -> dict[str, Any]:
rubric_id = str(rubric.get("id") or "rubric")
code = rubric.get("verifier_code")
if not isinstance(code, str) or not code.strip():
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": "rubric has no verifier_code",
}
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(compile(code, f"<{rubric_id}>", "exec"), namespace)
verify = namespace.get("verify")
if not callable(verify):
raise RuntimeError("verifier_code did not define verify()")
result = _coerce_result(verify(str(WORKDIR), str(external_services_path)))
return {"id": rubric_id, **result}
except Exception:
return {
"id": rubric_id,
"pass": False,
"score": 0.0,
"feedback": traceback.format_exc(),
}
def main() -> None:
rubrics_path = TESTS_DIR / "rubrics.json"
if not rubrics_path.is_file():
print("[sop-verifier] ERROR: rubrics.json not found", file=sys.stderr)
sys.exit(1)
rubrics = json.loads(rubrics_path.read_text())
if not isinstance(rubrics, list):
print("[sop-verifier] ERROR: rubrics.json must be a list", file=sys.stderr)
sys.exit(1)
with tempfile.TemporaryDirectory(prefix="sop-external-services-") as tmp:
compat_dir = Path(tmp)
_build_compat_external_services(compat_dir)
results = [_run_one(r, compat_dir) for r in rubrics]
total = len(results)
passed = sum(1 for r in results if r.get("pass"))
average_score = round(
sum(float(r.get("score", 0.0)) for r in results) / total,
4,
) if total else 0.0
print(f"[sop-verifier] {passed}/{total} rubrics passed; score={average_score:.2f}")
for result in results:
status = "PASS" if result.get("pass") else "FAIL"
feedback = str(result.get("feedback", "")).replace("\n", " ")[:500]
print(f" [{status}] {result.get('id')}: {feedback}")
output = {
"passed": passed == total,
"rubrics_passed": passed,
"rubrics_total": total,
"score": average_score,
"rubric_results": results,
}
(TESTS_DIR / "results.json").write_text(json.dumps(output, indent=2) + "\n")
VERIFIER_DIR.mkdir(parents=True, exist_ok=True)
(VERIFIER_DIR / "reward.txt").write_text(str(average_score))
if __name__ == "__main__":
main()
@@ -0,0 +1,13 @@
Hey, we've got a bit of a situation this morning and I could really use your help sorting it out. It's BD 3 of month-end close and everything that could go wrong seems to have landed on the same deal.
The Johnson deal from last week (SL-2026-0412) has turned into a perfect storm. First, we got a chargeback notice this morning from SafeGuard Warranty for $2,800 on the VSC. But that's not even the half of it. The deal's been sitting unfunded with Wells Fargo for six days now, especially since the contract is over $30K.
Then there's the trade payoff. We wired Chase a few days ago for what we thought was $11,250, but they applied $11,540. That's a $290 variance from our estimate.
Oh, and to top it all off, the daily cash deposit from yesterday is short by $180 on this same deal. Renee's already flagged it and said not to close the daily report until we figure it out.
On top of all that, the floorplan worksheet shows this unit as sold-not-paid and it's been accruing interest since delivery. We need to make sure we're handling that correctly too.
I need you to work through all of this following our Accounting SOP. Consult the SOP and the other files and resources as needed. Assume it's 9:00 AM on Monday, April 14, 2026. I need all the appropriate actions completed by end of day.
Save a summary of all actions taken to actions_summary.txt in the workspace, including the decision rationale for each escalation path you chose.
@@ -0,0 +1,3 @@
#!/bin/bash
pip install openpyxl pdfplumber python-docx 2>/dev/null
python /tests/sop_verifier.py