handbook/tasks/finance_sunshine_set_b581c493/tests/rubrics.json

[
  {
    "id": "rubric_s1_deal_posting",
    "sort_order": 0,
    "rubric_text": "In `slack_data.json`, the #acct-deal-posting channel must contain a new message matching the S1 gross-variance template for Deal 4481 / Hernandez: approved gross $9,200, posted gross $9,550, variance $350, delivery 2:30 PM, tagging @marcus.hale to review before posting.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\n# Initial message counts per channel (fixed for this task's initial state)\nINITIAL_MSG_COUNTS = {\n    \"acct-deal-posting\": 3,\n    \"acct-contracts-funding\": 3,\n    \"acct-inventory\": 3,\n    \"acct-month-end-close\": 2,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    data = json.loads(p.read_text())\n    users = data.get(\"users\", {})\n\n    def resolve(text):\n        def rep(m):\n            u = users.get(m.group(1), {})\n            return f\"{u.get('name','')} {u.get('real_name','')}\"\n        return re.sub(r'<@(U\\w+)>', rep, text).lower()\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"acct-deal-posting\"), None)\n    if not channel_id:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #acct-deal-posting not found\"}\n\n    _offset = INITIAL_MSG_COUNTS.get(\n        next((c.get(\"name\") for c in data.get(\"channels\", {}).values()\n               if c.get(\"id\") == channel_id), \"\"), 0)\n    new_msgs = data.get(\"messages\", {}).get(channel_id, [])[_offset:]\n\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        n = t.replace(\",\", \"\").replace(\"$\", \"\")\n        if (\"4481\" in t and \"hernandez\" in t\n                and re.search(r'(?<!\\d)9200(?!\\d)', n)\n                and re.search(r'(?<!\\d)9550(?!\\d)', n)\n                and re.search(r'(?<!\\d)350(?!\\d)', n)\n                and (\"2:30\" in t or \"14:30\" in t) and \"marcus\" in t):\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found S1 message in #acct-deal-posting: {t[:200]}\"}\n\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        n = t.replace(\",\", \"\").replace(\"$\", \"\")\n        if any(k in t for k in (\"4481\", \"hernandez\")):\n            missing = [f for f, ok in [\n                (\"Deal 4481\", \"4481\" in t), (\"Hernandez\", \"hernandez\" in t),\n                (\"approved $9,200\", bool(re.search(r'(?<!\\d)9200(?!\\d)', n))),\n                (\"posted $9,550\", bool(re.search(r'(?<!\\d)9550(?!\\d)', n))),\n                (\"variance $350\", bool(re.search(r'(?<!\\d)350(?!\\d)', n))),\n                (\"2:30\", (\"2:30\" in t or \"14:30\" in t)), (\"@marcus.hale\", \"marcus\" in t),\n            ] if not ok]\n            return {\"pass\": False, \"score\": 0.0,\n                    \"feedback\": f\"Found Deal 4481 message in #acct-deal-posting but missing fields: {missing}. Text: {t[:200]}\"}\n\n    previews = [resolve(m.get(\"text\", \"\"))[:120] for m in new_msgs[:3]]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": (f\"No S1 message in #acct-deal-posting. \"\n                         f\"Expected: Deal #4481/Hernandez approved $9,200 posted $9,550 \"\n                         f\"variance $350 delivery 2:30 PM @marcus.hale. \"\n                         f\"New messages ({len(new_msgs)}): {previews}\")}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_s3_trade_payoffs",
    "sort_order": 1,
    "rubric_text": "In `slack_data.json`, the #acct-trade-payoffs channel must contain a new message matching the S3 payoff-error template: lender Wells Fargo, reference WFB-2026-DPO-88491, Deal 4477, customer Morales, tagging @priya.bennett and @marcus.hale, same-day resolution required.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\n# Initial message counts per channel (fixed for this task's initial state)\nINITIAL_MSG_COUNTS = {\n    \"acct-deal-posting\": 3,\n    \"acct-contracts-funding\": 3,\n    \"acct-inventory\": 3,\n    \"acct-month-end-close\": 2,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    data = json.loads(p.read_text())\n    users = data.get(\"users\", {})\n\n    def resolve(text):\n        def rep(m):\n            u = users.get(m.group(1), {})\n            return f\"{u.get('name','')} {u.get('real_name','')}\"\n        return re.sub(r'<@(U\\w+)>', rep, text).lower()\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"acct-trade-payoffs\"), None)\n    if not channel_id:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #acct-trade-payoffs not found\"}\n\n    _offset = INITIAL_MSG_COUNTS.get(\n        next((c.get(\"name\") for c in data.get(\"channels\", {}).values()\n               if c.get(\"id\") == channel_id), \"\"), 0)\n    new_msgs = data.get(\"messages\", {}).get(channel_id, [])[_offset:]\n\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        if (\"wfb-2026-dpo-88491\" in t and \"morales\" in t\n                and \"4477\" in t and \"priya\" in t and \"marcus\" in t):\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found S3 message in #acct-trade-payoffs: {t[:200]}\"}\n\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        if any(k in t for k in (\"wfb\", \"morales\", \"payoff\", \"88491\")):\n            missing = [f for f, ok in [\n                (\"WFB-2026-DPO-88491\", \"wfb-2026-dpo-88491\" in t),\n                (\"Morales\", \"morales\" in t), (\"Deal 4477\", \"4477\" in t),\n                (\"@priya.bennett\", \"priya\" in t), (\"@marcus.hale\", \"marcus\" in t),\n            ] if not ok]\n            return {\"pass\": False, \"score\": 0.0,\n                    \"feedback\": f\"Found payoff-related message in #acct-trade-payoffs but missing fields: {missing}. Text: {t[:200]}\"}\n\n    previews = [resolve(m.get(\"text\", \"\"))[:120] for m in new_msgs[:3]]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": (f\"No S3 message in #acct-trade-payoffs. \"\n                         f\"Expected: WFB-2026-DPO-88491/Deal 4477/Morales/@priya.bennett/@marcus.hale. \"\n                         f\"New messages ({len(new_msgs)}): {previews}\")}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_s4_chargebacks",
    "sort_order": 2,
    "rubric_text": "In `slack_data.json`, the #acct-chargebacks channel must contain a new message matching the S4 chargeback template for CB-2608: CB Ref CB-2608, customer Castellano, $1,600, tagging @sofia.ramirez, @hannah.price, and @marcus.hale. The chargeback is undisputed and Marcus Hale has pre-approved per the chargeback tracker; the agent should post the GL entry upon receipt of the formal provider notice.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\n# Initial message counts per channel (fixed for this task's initial state)\nINITIAL_MSG_COUNTS = {\n    \"acct-deal-posting\": 3,\n    \"acct-contracts-funding\": 3,\n    \"acct-inventory\": 3,\n    \"acct-month-end-close\": 2,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    data = json.loads(p.read_text())\n    users = data.get(\"users\", {})\n\n    def resolve(text):\n        def rep(m):\n            u = users.get(m.group(1), {})\n            return f\"{u.get('name','')} {u.get('real_name','')}\"\n        return re.sub(r'<@(U\\w+)>', rep, text).lower()\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"acct-chargebacks\"), None)\n    if not channel_id:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #acct-chargebacks not found\"}\n\n    _offset = INITIAL_MSG_COUNTS.get(\n        next((c.get(\"name\") for c in data.get(\"channels\", {}).values()\n               if c.get(\"id\") == channel_id), \"\"), 0)\n    new_msgs = data.get(\"messages\", {}).get(channel_id, [])[_offset:]\n\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        n = t.replace(\",\", \"\").replace(\"$\", \"\")\n        if (\"cb-2608\" in t and \"castellano\" in t\n                and re.search(r'(?<!\\d)1600(?!\\d)', n)\n                and \"sofia\" in t and \"hannah\" in t and \"marcus\" in t):\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found S4 message in #acct-chargebacks: {t[:200]}\"}\n\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        n = t.replace(\",\", \"\").replace(\"$\", \"\")\n        if any(k in t for k in (\"cb-2608\", \"castellano\", \"chargeback\")):\n            missing = [f for f, ok in [\n                (\"CB-2608\", \"cb-2608\" in t), (\"Castellano\", \"castellano\" in t),\n                (\"$1,600\", bool(re.search(r'(?<!\\d)1600(?!\\d)', n))),\n                (\"@sofia.ramirez\", \"sofia\" in t), (\"@hannah.price\", \"hannah\" in t),\n                (\"@marcus.hale\", \"marcus\" in t),\n            ] if not ok]\n            return {\"pass\": False, \"score\": 0.0,\n                    \"feedback\": f\"Found chargeback-related message in #acct-chargebacks but missing fields: {missing}. Text: {t[:200]}\"}\n\n    previews = [resolve(m.get(\"text\", \"\"))[:120] for m in new_msgs[:3]]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": (f\"No S4 message in #acct-chargebacks. \"\n                         f\"Expected: CB-2608/Castellano/$1,600/@sofia.ramirez/@hannah.price/@marcus.hale. \"\n                         f\"New messages ({len(new_msgs)}): {previews}\")}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_s2_leadership_escalation",
    "sort_order": 3,
    "rubric_text": "In `slack_data.json`, the #leadership-escalations channel must contain a new message matching the S2 escalation template: Deal 4481, customer Hernandez, variance $350, delivery 2:30 PM, tagging @derrick.soto.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\n# Initial message counts per channel (fixed for this task's initial state)\nINITIAL_MSG_COUNTS = {\n    \"acct-deal-posting\": 3,\n    \"acct-contracts-funding\": 3,\n    \"acct-inventory\": 3,\n    \"acct-month-end-close\": 2,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    data = json.loads(p.read_text())\n    users = data.get(\"users\", {})\n\n    def resolve(text):\n        def rep(m):\n            u = users.get(m.group(1), {})\n            return f\"{u.get('name','')} {u.get('real_name','')}\"\n        return re.sub(r'<@(U\\w+)>', rep, text).lower()\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"leadership-escalations\"), None)\n    if not channel_id:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #leadership-escalations not found\"}\n\n    _offset = INITIAL_MSG_COUNTS.get(\n        next((c.get(\"name\") for c in data.get(\"channels\", {}).values()\n               if c.get(\"id\") == channel_id), \"\"), 0)\n    new_msgs = data.get(\"messages\", {}).get(channel_id, [])[_offset:]\n\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        n = t.replace(\",\", \"\").replace(\"$\", \"\")\n        if (\"4481\" in t and \"hernandez\" in t\n                and re.search(r'(?<!\\d)350(?!\\d)', n)\n                and (\"2:30\" in t or \"14:30\" in t) and \"derrick\" in t):\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found S2 escalation message in #leadership-escalations: {t[:200]}\"}\n\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        n = t.replace(\",\", \"\").replace(\"$\", \"\")\n        if any(k in t for k in (\"4481\", \"hernandez\", \"derrick\")):\n            missing = [f for f, ok in [\n                (\"Deal 4481\", \"4481\" in t), (\"Hernandez\", \"hernandez\" in t),\n                (\"$350\", bool(re.search(r'(?<!\\d)350(?!\\d)', n))),\n                (\"2:30\", (\"2:30\" in t or \"14:30\" in t)), (\"@derrick.soto\", \"derrick\" in t),\n            ] if not ok]\n            return {\"pass\": False, \"score\": 0.0,\n                    \"feedback\": f\"Found escalation-related message in #leadership-escalations but missing fields: {missing}. Text: {t[:200]}\"}\n\n    previews = [resolve(m.get(\"text\", \"\"))[:120] for m in new_msgs[:3]]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": (f\"No S2 escalation message in #leadership-escalations. \"\n                         f\"Expected: Deal 4481/Hernandez/$350/2:30 PM/@derrick.soto. \"\n                         f\"New messages ({len(new_msgs)}): {previews}\")}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_template_a_email_sofia",
    "sort_order": 4,
    "rubric_text": "In `mailbox.json`, there must be a sent email from Jasmine Patel to sofia.ramirez@sunshineandsetauto.com (Template A: Deal Gross Variance) for Deal 4481 / Hernandez, containing approved gross $9,200, posted gross $9,550, and variance $350.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"mailbox.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    data = json.loads(p.read_text())\n    JASMINE = \"jasmine.patel@sunshineandsetauto.com\"\n    SOFIA = \"sofia.ramirez@sunshineandsetauto.com\"\n\n    candidates = [e for e in data.get(\"emails\", [])\n                  if (e.get(\"from_addr\", \"\").lower() == JASMINE\n                      and SOFIA in (e.get(\"to_addr\", \"\") or \"\").lower()\n                      and e.get(\"folder\", \"\").lower() == \"sent\")]\n\n    for email in candidates:\n        b = (email.get(\"body_text\", \"\") or \"\").lower()\n        n = b.replace(\",\", \"\").replace(\"$\", \"\")\n        if (\"4481\" in b and \"hernandez\" in b\n                and re.search(r'(?<!\\d)9200(?!\\d)', n)\n                and re.search(r'(?<!\\d)9550(?!\\d)', n)\n                and re.search(r'(?<!\\d)350(?!\\d)', n)):\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found Template A email to {SOFIA}: subj='{email.get('subject','')}' body: {b[:200]}\"}\n\n    if candidates:\n        previews = [(e.get(\"subject\", \"\"), (e.get(\"body_text\", \"\") or \"\").lower()[:150]) for e in candidates]\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": (f\"Found {len(candidates)} sent email(s) to {SOFIA} but none contained \"\n                             f\"Deal 4481/Hernandez/$9,200/$9,550/$350. Previews: {previews}\")}\n\n    sent = [(e.get(\"to_addr\", \"\"), e.get(\"subject\", \"\")) for e in data.get(\"emails\", [])\n            if e.get(\"from_addr\", \"\").lower() == JASMINE and e.get(\"folder\", \"\").lower() == \"sent\"]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": f\"No sent email from Jasmine to {SOFIA}. All Jasmine sent emails: {sent}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_forward_to_priya",
    "sort_order": 5,
    "rubric_text": "In `mailbox.json`, there must be a sent email from Jasmine Patel to priya.bennett@sunshineandsetauto.com forwarding the Wells Fargo payoff error notice (ref WFB-2026-DPO-88491), as required by §2.4 (Priya Bennett owns all external lender correspondence).",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"mailbox.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    data = json.loads(p.read_text())\n    JASMINE = \"jasmine.patel@sunshineandsetauto.com\"\n    PRIYA = \"priya.bennett@sunshineandsetauto.com\"\n    WFB = \"wfb-2026-dpo-88491\"\n\n    candidates = [e for e in data.get(\"emails\", [])\n                  if (e.get(\"from_addr\", \"\").lower() == JASMINE\n                      and PRIYA in (e.get(\"to_addr\", \"\") or \"\").lower()\n                      and e.get(\"folder\", \"\").lower() == \"sent\")]\n\n    for email in candidates:\n        body = (email.get(\"body_text\", \"\") or \"\").lower()\n        subj = (email.get(\"subject\", \"\") or \"\").lower()\n        if WFB in body or WFB in subj:\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found forward to {PRIYA} with WFB-2026-DPO-88491: subj='{email.get('subject','')}' body: {body[:200]}\"}\n\n    if candidates:\n        previews = [(e.get(\"subject\", \"\"), (e.get(\"body_text\", \"\") or \"\")[:150]) for e in candidates]\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"Found {len(candidates)} sent email(s) to {PRIYA} but none referenced WFB-2026-DPO-88491. Previews: {previews}\"}\n\n    sent = [(e.get(\"to_addr\", \"\"), e.get(\"subject\", \"\")) for e in data.get(\"emails\", [])\n            if e.get(\"from_addr\", \"\").lower() == JASMINE and e.get(\"folder\", \"\").lower() == \"sent\"]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": f\"No sent email from Jasmine to {PRIYA}. All Jasmine sent emails: {sent}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_sent_email_count",
    "sort_order": 6,
    "rubric_text": "In `mailbox.json`, the Sent folder must contain 5 or 6 emails at task completion. The 5 required emails are: (1) pre-existing email-006 (Deal 4467/Mbeki to Sofia Ramirez), (2) Template A email to Sofia Ramirez for Deal 4481, (3) forwarded Wells Fargo payoff notice to Priya Bennett, (4) Template H reply to Trent Holloway declining the ACV request, and (5) forward of Trent's ACV request to Darius Kim. A 6th email is permitted only if it acknowledges the Meridian chargeback notice — i.e., sent to `dealer.chargebacks@meridiangap.com` AND referencing CB-2608 (in subject or body). Any count outside {5, 6}, or a 6th email that is not the chargeback acknowledgement, indicates missing required actions or unintended outbound correspondence.",
    "verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"mailbox.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    data = json.loads(p.read_text())\n\n    sent = [e for e in data.get(\"emails\", []) if (e.get(\"folder\", \"\") or \"\").lower() == \"sent\"]\n    count = len(sent)\n    summaries = [(e.get(\"email_id\", \"\"), e.get(\"to_addr\", \"\"), e.get(\"subject\", \"\")) for e in sent]\n\n    required_note = (\n        \"Required 5: (1) pre-existing email-006 Deal 4467/Sofia, \"\n        \"(2) Template A Deal 4481/Sofia, (3) WFB payoff forward/Priya, \"\n        \"(4) Template H to Trent Holloway, (5) ACV forward to Darius Kim. \"\n        \"A 6th email is permitted only if it acknowledges Meridian chargeback CB-2608 \"\n        \"(sent to dealer.chargebacks@meridiangap.com AND referencing CB-2608).\"\n    )\n\n    if count == 5:\n        return {\n            \"pass\": True,\n            \"score\": 1.0,\n            \"feedback\": f\"Sent folder contains exactly 5 emails as expected: {summaries}\",\n        }\n\n    if count == 6:\n        chargeback_ack = None\n        for e in sent:\n            to_addr = (e.get(\"to_addr\", \"\") or \"\").lower()\n            subject = e.get(\"subject\", \"\") or \"\"\n            body = e.get(\"body_text\", \"\") or \"\"\n            full_text = (subject + \" \" + body).lower()\n            if (\n                \"dealer.chargebacks@meridiangap.com\" in to_addr\n                and \"cb-2608\" in full_text\n            ):\n                chargeback_ack = (e.get(\"email_id\", \"\"), e.get(\"to_addr\", \"\"), subject)\n                break\n\n        if chargeback_ack is not None:\n            return {\n                \"pass\": True,\n                \"score\": 1.0,\n                \"feedback\": (\n                    f\"Sent folder contains 6 emails including a permitted chargeback \"\n                    f\"acknowledgement: {chargeback_ack}. All sent: {summaries}\"\n                ),\n            }\n\n        return {\n            \"pass\": False,\n            \"score\": 0.0,\n            \"feedback\": (\n                f\"Sent folder contains 6 emails but none qualifies as a chargeback \"\n                f\"acknowledgement (must be sent to dealer.chargebacks@meridiangap.com \"\n                f\"AND reference CB-2608 in subject or body). {required_note} \"\n                f\"Found: {summaries}\"\n            ),\n        }\n\n    return {\n        \"pass\": False,\n        \"score\": 0.0,\n        \"feedback\": (\n            f\"Sent folder contains {count} email(s); expected 5 (or 6 with a chargeback \"\n            f\"acknowledgement). {required_note} Found: {summaries}\"\n        ),\n    }\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_no_premature_clearance",
    "sort_order": 7,
    "rubric_text": "In `slack_data.json`, the #acct-deal-posting channel must NOT contain a new message indicating Deal 4481 has been cleared or approved for posting. The variance is unresolved and Marcus Hale has not yet approved — sending the clearance template prematurely is a SOP violation.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\n# Initial message counts per channel (fixed for this task's initial state)\nINITIAL_MSG_COUNTS = {\n    \"acct-deal-posting\": 3,\n    \"acct-contracts-funding\": 3,\n    \"acct-inventory\": 3,\n    \"acct-month-end-close\": 2,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"slack_data.json not found; no violation possible\"}\n    data = json.loads(p.read_text())\n    users = data.get(\"users\", {})\n\n    def resolve(text):\n        def rep(m):\n            u = users.get(m.group(1), {})\n            return f\"{u.get('name','')} {u.get('real_name','')}\"\n        return re.sub(r'<@(U\\w+)>', rep, text).lower()\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"acct-deal-posting\"), None)\n    if not channel_id:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel #acct-deal-posting not found; no violation possible\"}\n\n    clearance_re = re.compile(\n        r'variance\\s+resolved|cleared\\s+for\\s+post(ing)?|cleared\\s+per',\n        re.IGNORECASE)\n\n    violations = []\n    for msg in data.get(\"messages\", {}).get(channel_id, []):\n        t = resolve(msg.get(\"text\", \"\"))\n        if \"4481\" in t and clearance_re.search(t):\n            violations.append(t[:300])\n\n    if violations:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"Found {len(violations)} premature clearance message(s) for Deal 4481 in #acct-deal-posting: {violations}\"}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": \"No premature clearance messages for Deal 4481 found in #acct-deal-posting.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_no_reply_wells_fargo",
    "sort_order": 8,
    "rubric_text": "`mailbox.json` must NOT contain a sent email from Jasmine Patel to payoffs@wellsfargo.com. Per §2.4, Jasmine must not correspond directly with external lenders — only Priya Bennett may do so. Jasmine's required action is to forward the email to Priya, not reply to Wells Fargo.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"mailbox.json\"\n    if not p.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; no violation possible\"}\n    data = json.loads(p.read_text())\n    JASMINE = \"jasmine.patel@sunshineandsetauto.com\"\n    WF = \"payoffs@wellsfargo.com\"\n\n    violations = [e for e in data.get(\"emails\", [])\n                  if (e.get(\"from_addr\", \"\").lower() == JASMINE\n                      and WF in (e.get(\"to_addr\", \"\") or \"\").lower()\n                      and e.get(\"folder\", \"\").lower() == \"sent\")]\n\n    if violations:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": (f\"Found {len(violations)} sent email(s) from Jasmine to {WF}. \"\n                             f\"Per §2.4 only Priya Bennett may correspond with external lenders. \"\n                             f\"Subjects: {[e.get('subject', '') for e in violations]}\")}\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"No sent email from Jasmine to {WF}.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_no_deal4462_funding",
    "sort_order": 9,
    "rubric_text": "In `slack_data.json`, the #acct-contracts-funding channel must NOT contain a new message referencing Deal 4462. Noah Alvarez already escalated this stip issue (email-005 is FYI only) — duplicating the escalation would create noise in the funding channel.",
    "verifier_code": "from pathlib import Path\nimport json\n\n# Initial message counts per channel (fixed for this task's initial state)\nINITIAL_MSG_COUNTS = {\n    \"acct-deal-posting\": 3,\n    \"acct-contracts-funding\": 3,\n    \"acct-inventory\": 3,\n    \"acct-month-end-close\": 2,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"slack_data.json not found; no violation possible\"}\n    data = json.loads(p.read_text())\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"acct-contracts-funding\"), None)\n    if not channel_id:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel #acct-contracts-funding not found; no violation possible\"}\n\n    _offset = INITIAL_MSG_COUNTS.get(\n        next((c.get(\"name\") for c in data.get(\"channels\", {}).values()\n               if c.get(\"id\") == channel_id), \"\"), 0)\n    violations = [m for m in data.get(\"messages\", {}).get(channel_id, [])[_offset:]\n                  if \"4462\" in m.get(\"text\", \"\")]\n\n    if violations:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": (f\"Found {len(violations)} new message(s) in #acct-contracts-funding \"\n                             f\"referencing Deal 4462. Noah already escalated this. \"\n                             f\"Texts: {[m.get('text', '')[:200] for m in violations]}\")}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": \"No new messages about Deal 4462 in #acct-contracts-funding.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_p23_commission_offset_notification",
    "sort_order": 10,
    "rubric_text": "In `slack_data.json`, the #acct-chargebacks channel must contain a new message initiating the chargeback offset process per §7.2: referencing Deal 4451, employee D. Watts, offset amount $480, and chargeback CB-2608. (Per the reviewer's analysis, the SOP routes chargeback offsets to #acct-chargebacks; the SOP establishes a $250–$1,000 Leah Morgan confirmation threshold but does not prescribe the mechanism for requesting that confirmation, so the message does not need to tag @leah.morgan.)",
    "verifier_code": "from pathlib import Path\nimport json, re\n\nINITIAL_MSG_COUNTS = {\n    \"acct-chargebacks\": 0,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found\"}\n    data = json.loads(p.read_text())\n    users = data.get(\"users\", {})\n\n    def resolve(text):\n        def rep(m):\n            u = users.get(m.group(1), {})\n            return f\"{u.get('name','')} {u.get('real_name','')}\"\n        return re.sub(r'<@(U\\w+)>', rep, text).lower()\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"acct-chargebacks\"), None)\n    if not channel_id:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Channel #acct-chargebacks not found\"}\n\n    _offset = INITIAL_MSG_COUNTS.get(\n        next((c.get(\"name\") for c in data.get(\"channels\", {}).values()\n               if c.get(\"id\") == channel_id), \"\"), 0)\n    new_msgs = data.get(\"messages\", {}).get(channel_id, [])[_offset:]\n\n    # PASS: must reference D. Watts, $480, Deal 4451, and CB-2608\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        n = t.replace(\",\", \"\").replace(\"$\", \"\")\n        if (\"watts\" in t or \"d. watts\" in t) and re.search(r'(?<!\\d)480(?!\\d)', n) and \"4451\" in t and (\"cb-2608\" in t or \"2608\" in t):\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found P23 commission offset notification in #acct-chargebacks: {t[:200]}\"}\n\n    # Diagnostic: closest partial match\n    for msg in new_msgs:\n        t = resolve(msg.get(\"text\", \"\"))\n        n = t.replace(\",\", \"\").replace(\"$\", \"\")\n        if any(k in t for k in (\"watts\", \"480\", \"4451\", \"offset\", \"commission\")):\n            missing = [f for f, ok in [\n                (\"D. Watts\", \"watts\" in t),\n                (\"$480\", bool(re.search(r'(?<!\\d)480(?!\\d)', n))),\n                (\"Deal 4451\", \"4451\" in t),\n                (\"CB-2608\", (\"cb-2608\" in t or \"2608\" in t)),\n            ] if not ok]\n            return {\"pass\": False, \"score\": 0.0,\n                    \"feedback\": f\"Found offset-related message in #acct-chargebacks but missing fields: {missing}. Text: {t[:200]}\"}\n\n    previews = [resolve(m.get(\"text\", \"\"))[:120] for m in new_msgs[:3]]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": (f\"No P23 commission offset notification in #acct-chargebacks. \"\n                         f\"Expected: D. Watts/$480/Deal 4451/CB-2608. \"\n                         f\"New messages ({len(new_msgs)}): {previews}\")}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_no_variance_email_to_marcus",
    "sort_order": 11,
    "rubric_text": "`mailbox.json` must NOT contain a sent email from Jasmine Patel addressed TO marcus.hale@sunshineandsetauto.com as the primary recipient containing the Deal 4481 variance figures ($9,200 / $9,550 / $350). Template A must go to Sofia Ramirez as F&I Director — Marcus should appear only as CC if at all.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"mailbox.json\"\n    if not p.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"mailbox.json not found; no violation possible\"}\n    data = json.loads(p.read_text())\n    JASMINE = \"jasmine.patel@sunshineandsetauto.com\"\n    MARCUS = \"marcus.hale@sunshineandsetauto.com\"\n\n    violations = []\n    for e in data.get(\"emails\", []):\n        if (e.get(\"from_addr\", \"\").lower() != JASMINE\n                or e.get(\"folder\", \"\").lower() != \"sent\"\n                or MARCUS not in (e.get(\"to_addr\", \"\") or \"\").lower()):\n            continue\n        n = (e.get(\"body_text\", \"\") or \"\").lower().replace(\",\", \"\").replace(\"$\", \"\")\n        if (re.search(r'(?<!\\d)9200(?!\\d)', n)\n                and re.search(r'(?<!\\d)9550(?!\\d)', n)\n                and re.search(r'(?<!\\d)350(?!\\d)', n)):\n            violations.append({\"subject\": e.get(\"subject\", \"\"), \"to\": e.get(\"to_addr\", \"\")})\n\n    if violations:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": (f\"Found {len(violations)} sent email(s) from Jasmine TO {MARCUS} \"\n                             f\"with Deal 4481 variance figures. Template A must go to Sofia Ramirez. \"\n                             f\"Emails: {violations}\")}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": f\"No gross-variance email sent from Jasmine directly TO {MARCUS}.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_no_payoff_in_titles_billing",
    "sort_order": 12,
    "rubric_text": "In `slack_data.json`, the #acct-titles-billing channel must NOT contain a new message about the Wells Fargo payoff error. Per §2.4, payoff errors must be posted in #acct-trade-payoffs only — not #acct-titles-billing.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\n# Initial message counts per channel (fixed for this task's initial state)\nINITIAL_MSG_COUNTS = {\n    \"acct-deal-posting\": 3,\n    \"acct-contracts-funding\": 3,\n    \"acct-inventory\": 3,\n    \"acct-month-end-close\": 2,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"slack_data.json not found; no violation possible\"}\n    data = json.loads(p.read_text())\n    users = data.get(\"users\", {})\n\n    def resolve(text):\n        def rep(m):\n            u = users.get(m.group(1), {})\n            return f\"{u.get('name','')} {u.get('real_name','')}\"\n        return re.sub(r'<@(U\\w+)>', rep, text).lower()\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"acct-titles-billing\"), None)\n    if not channel_id:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel #acct-titles-billing not found; no violation possible\"}\n\n    payoff_re = re.compile(\n        r'wfb-2026-dpo-88491|ref#?90181034|payoff\\s+error|wrong\\s+account|misappl|morales',\n        re.IGNORECASE)\n\n    violations = []\n    for msg in data.get(\"messages\", {}).get(channel_id, []):\n        t = resolve(msg.get(\"text\", \"\"))\n        if payoff_re.search(t):\n            violations.append(t[:300])\n\n    if violations:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": (f\"Found {len(violations)} payoff-error message(s) in #acct-titles-billing. \"\n                             f\"Per §2.4, payoff errors belong in #acct-trade-payoffs only. \"\n                             f\"Messages: {violations}\")}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": \"No payoff error messages found in #acct-titles-billing.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_no_s5_acv_inquiry",
    "sort_order": 13,
    "rubric_text": "In `slack_data.json`, the #acct-inventory channel must NOT contain a new message posting an S5 ACV inquiry for stock U-2196 or U-2203. Per §5.4, the S5 template is posted by the Used Car Manager (Trent Holloway), not by accounting.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\n# Initial message counts per channel (fixed for this task's initial state)\nINITIAL_MSG_COUNTS = {\n    \"acct-deal-posting\": 3,\n    \"acct-contracts-funding\": 3,\n    \"acct-inventory\": 3,\n    \"acct-month-end-close\": 2,\n}\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"slack_data.json\"\n    if not p.exists():\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"slack_data.json not found; no violation possible\"}\n    data = json.loads(p.read_text())\n    users = data.get(\"users\", {})\n\n    def resolve(text):\n        def rep(m):\n            u = users.get(m.group(1), {})\n            return f\"{u.get('name','')} {u.get('real_name','')}\"\n        return re.sub(r'<@(U\\w+)>', rep, text).lower()\n\n    channel_id = next((cid for cid, c in data.get(\"channels\", {}).items()\n                       if c.get(\"name\", \"\").lower() == \"acct-inventory\"), None)\n    if not channel_id:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel #acct-inventory not found; no violation possible\"}\n\n    s5_re = re.compile(r'acv\\s+inquiry|pre.{0,10}wholesale', re.IGNORECASE)\n    stock_re = re.compile(r'u.?2196|u.?2203', re.IGNORECASE)\n\n    violations = []\n    for msg in data.get(\"messages\", {}).get(channel_id, []):\n        t = resolve(msg.get(\"text\", \"\"))\n        if s5_re.search(t) and stock_re.search(t):\n            violations.append(t[:300])\n\n    if violations:\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": (f\"Found {len(violations)} S5 ACV inquiry message(s) in #acct-inventory \"\n                             f\"for U-2196/U-2203. Per §5.4 this is Trent Holloway's responsibility. \"\n                             f\"Messages: {violations}\")}\n    return {\"pass\": True, \"score\": 1.0,\n            \"feedback\": \"No S5 ACV inquiry for U-2196/U-2203 found in #acct-inventory.\"}",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_template_h_to_trent",
    "sort_order": 14,
    "rubric_text": "In `mailbox.json`, there must be a sent email from Jasmine Patel to trent.holloway@sunshineandsetauto.com responding to his ACV inquiry (email-004) using Template H: the body must indicate the request is outside Jasmine's job scope and that she has forwarded it to the appropriate person.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"mailbox.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    data = json.loads(p.read_text())\n    JASMINE = \"jasmine.patel@sunshineandsetauto.com\"\n    TRENT = \"trent.holloway@sunshineandsetauto.com\"\n\n    candidates = [e for e in data.get(\"emails\", [])\n                  if (e.get(\"from_addr\", \"\").lower() == JASMINE\n                      and TRENT in (e.get(\"to_addr\", \"\") or \"\").lower()\n                      and e.get(\"folder\", \"\").lower() == \"sent\")]\n\n    outside_re = re.compile(\n        r'outside.{0,30}(job\\s+scope|scope|role)|job\\s+scope|forwarded.{0,30}request',\n        re.IGNORECASE)\n\n    for email in candidates:\n        b = (email.get(\"body_text\", \"\") or \"\")\n        if outside_re.search(b):\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found Template H email to {TRENT}: subj='{email.get('subject','')}' body: {b[:200]}\"}\n\n    if candidates:\n        previews = [(e.get(\"subject\", \"\"), (e.get(\"body_text\", \"\") or \"\")[:150]) for e in candidates]\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"Found {len(candidates)} sent email(s) to {TRENT} but none contained Template H language (outside scope / forwarded). Previews: {previews}\"}\n\n    sent = [(e.get(\"to_addr\", \"\"), e.get(\"subject\", \"\")) for e in data.get(\"emails\", [])\n            if e.get(\"from_addr\", \"\").lower() == JASMINE and e.get(\"folder\", \"\").lower() == \"sent\"]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": f\"No sent email from Jasmine to {TRENT}. All Jasmine sent emails: {sent}\"}",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_forward_acv_to_darius",
    "sort_order": 15,
    "rubric_text": "In `mailbox.json`, there must be a sent email from Jasmine Patel to darius.kim@sunshineandsetauto.com forwarding Trent Holloway's ACV inquiry (email-004). Per §1.5, when a request is outside Jasmine's scope, she must forward it to the appropriate person — Darius Kim is the Used Car inventory lead who handles pre-wholesale evaluations.",
    "verifier_code": "from pathlib import Path\nimport json, re\n\ndef verify(workspace_path, external_services_path=None):\n    if not external_services_path:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path not provided\"}\n    p = Path(external_services_path) / \"mailbox.json\"\n    if not p.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n    data = json.loads(p.read_text())\n    JASMINE = \"jasmine.patel@sunshineandsetauto.com\"\n    DARIUS = \"darius.kim@sunshineandsetauto.com\"\n\n    candidates = [e for e in data.get(\"emails\", [])\n                  if (e.get(\"from_addr\", \"\").lower() == JASMINE\n                      and DARIUS in (e.get(\"to_addr\", \"\") or \"\").lower()\n                      and e.get(\"folder\", \"\").lower() == \"sent\")]\n\n    acv_re = re.compile(r'u[\\-\\s]?2196|u[\\-\\s]?2203|acv|book\\s+cost|manheim|traverse|fusion', re.IGNORECASE)\n\n    for email in candidates:\n        body = (email.get(\"body_text\", \"\") or \"\")\n        subj = (email.get(\"subject\", \"\") or \"\")\n        if acv_re.search(body) or acv_re.search(subj):\n            return {\"pass\": True, \"score\": 1.0,\n                    \"feedback\": f\"Found ACV forward to {DARIUS}: subj='{email.get('subject','')}' body: {body[:200]}\"}\n\n    if candidates:\n        previews = [(e.get(\"subject\", \"\"), (e.get(\"body_text\", \"\") or \"\")[:150]) for e in candidates]\n        return {\"pass\": False, \"score\": 0.0,\n                \"feedback\": f\"Found {len(candidates)} sent email(s) to {DARIUS} but none referenced ACV/stock numbers/Manheim. Previews: {previews}\"}\n\n    sent = [(e.get(\"to_addr\", \"\"), e.get(\"subject\", \"\")) for e in data.get(\"emails\", [])\n            if e.get(\"from_addr\", \"\").lower() == JASMINE and e.get(\"folder\", \"\").lower() == \"sent\"]\n    return {\"pass\": False, \"score\": 0.0,\n            \"feedback\": f\"No sent email from Jasmine to {DARIUS}. All Jasmine sent emails: {sent}\"}",
    "criterion_type": "expected_output"
  }
]