handbook/tasks/finance_meridian_partners_cc2fc143/tests/rubrics.json

[
  {
    "id": "53c53e31-7fa0-4136-9885-e8eb58789e27",
    "sort_order": 0,
    "rubric_text": "In file `ap_ledger.xlsx`, row 13 must contain invoice data for GLFD-0441 with vendor V-00009 (Great Lakes Furniture & Design Co.), invoice date 2026-03-20, received date 2026-03-24, due date 2026-04-19, type Non-PO, amount $7,900.00, GL code 1000-100-6200, and status Open through the 'Status' column",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n    if not xlsx_files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n    ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n    target_files = ledger_files if ledger_files else xlsx_files\n\n    for fpath in target_files:\n        try:\n            wb = openpyxl.load_workbook(fpath, data_only=True)\n            ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n            # Find Status column from header row\n            status_col = None\n            for col in range(1, ws.max_column + 1):\n                hdr = ws.cell(row=1, column=col).value\n                if hdr is not None and str(hdr).strip().lower() == \"status\":\n                    status_col = col\n                    break\n\n            row_data = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[13]]\n            row_str = ' '.join(row_data).lower()\n\n            has_invoice_id = 'glfd-0441' in row_str\n            has_vendor_id = 'v-00009' in row_str\n            has_vendor_name = 'great lakes furniture & design co' in row_str\n            has_invoice_date = '2026-03-20' in row_str\n            has_received_date = '2026-03-24' in row_str\n            has_due_date = '2026-04-19' in row_str\n            has_non_po = 'non-po' in row_str or 'non po' in row_str\n            amount_found = '7,900' in row_str or '7900' in row_str\n            has_gl = '1000-100-6200' in row_str\n\n            if status_col is not None:\n                status_val = ws.cell(row=13, column=status_col).value\n                has_open = status_val is not None and str(status_val).strip().lower() == 'open'\n            else:\n                has_open = 'open' in row_str  # fallback if header not found\n\n            checks = {\n                'invoice_id GLFD-0441': has_invoice_id,\n                'vendor_id V-00009': has_vendor_id,\n                'vendor_name Great Lakes': has_vendor_name,\n                'invoice_date 2026-03-20': has_invoice_date,\n                'received_date 2026-03-24': has_received_date,\n                'due_date 2026-04-19': has_due_date,\n                'type Non-PO': has_non_po,\n                'amount $7,900.00': amount_found,\n                'gl_code 1000-100-6200': has_gl,\n                'status Open': has_open\n            }\n\n            failed = [k for k, v in checks.items() if not v]\n            passed = [k for k, v in checks.items() if v]\n\n            if not failed:\n                return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 13 in {fpath.name} sheet '{ws.title}' contains all required fields: {', '.join(passed)}\"}\n        except Exception as e:\n            continue\n\n    # Partial feedback pass\n    for fpath in target_files:\n        try:\n            wb = openpyxl.load_workbook(fpath, data_only=True)\n            ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n            status_col = None\n            for col in range(1, ws.max_column + 1):\n                hdr = ws.cell(row=1, column=col).value\n                if hdr is not None and str(hdr).strip().lower() == \"status\":\n                    status_col = col\n                    break\n\n            row_data = [str(cell.value).strip() if cell.value is not None else '' for cell in ws[13]]\n            row_str = ' '.join(row_data).lower()\n\n            if status_col is not None:\n                status_val = ws.cell(row=13, column=status_col).value\n                has_open = status_val is not None and str(status_val).strip().lower() == 'open'\n            else:\n                has_open = 'open' in row_str\n\n            checks = {\n                'invoice_id GLFD-0441': 'glfd-0441' in row_str,\n                'vendor_id V-00009': 'v-00009' in row_str,\n                'vendor_name Great Lakes': 'great lakes' in row_str,\n                'invoice_date 2026-03-20': '2026-03-20' in row_str,\n                'received_date 2026-03-24': '2026-03-24' in row_str,\n                'due_date 2026-04-19': '2026-04-19' in row_str,\n                'type Non-PO': 'non-po' in row_str or 'non po' in row_str,\n                'amount $7,900.00': '7,900' in row_str or '7900' in row_str,\n                'gl_code 1000-100-6200': '1000-100-6200' in row_str,\n                'status Open': has_open\n            }\n            failed = [k for k, v in checks.items() if not v]\n            passed = [k for k, v in checks.items() if v]\n            if passed:\n                return {\"pass\": False, \"score\": len(passed)/len(checks), \"feedback\": f\"Row 13 in {fpath.name} missing: {', '.join(failed)}. Found: {', '.join(passed)}. Row content: {row_data}\"}\n        except Exception:\n            continue\n\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read ap_ledger.xlsx with valid row 13 data\"}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "1bcd6b5d-4a16-42e7-9464-4a8a2c676eda",
    "sort_order": 1,
    "rubric_text": "In mailbox.json (Drafts folder), exactly one draft email must exist addressed to m.chen@meridianpartners.com and r.whitfield@meridianpartners.com with CC to ap@meridianpartners.com, subject 'AP APPROVAL REQUIRED: Great Lakes Furniture & Design Co. - $7,900.00 - Invoice GLFD-0441', and body containing only the following information and no other information: 'A non-PO invoice requires your approval before payment can be processed. Vendor: Great Lakes Furniture & Design Co. (V-00009). Invoice: GLFD-0441, dated 2026-03-20. Amount: $7,900.00. Description: Executive conference table, 10-person, walnut finish. GL Code: 1000-100-6200. Please reply APPROVED or REJECTED with reason. If no response is received within 5 business days, the invoice will be placed on APPR hold and the vendor will be notified of the delay.'",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\nimport unicodedata\n\ndef normalize(s):\n    s = unicodedata.normalize('NFKC', s)\n    s = s.replace('\\u2014', '-').replace('\\u2013', '-')\n    s = s.replace('\\u2019', \"'\").replace('\\u2018', \"'\")\n    s = s.replace('\\u201c', '\"').replace('\\u201d', '\"')\n    s = re.sub(r'[();:!?]', '', s)\n    s = re.sub(r'[,.](?!\\d)', '', s)\n    s = s.lower()\n    s = re.sub(r'\\s+', ' ', s).strip()\n    return s\n\nEXPECTED_SUBJECT = \"AP APPROVAL REQUIRED: Great Lakes Furniture & Design Co. - $7,900.00 - Invoice GLFD-0441\"\n\nEXPECTED_BODY = (\n    \"A non-PO invoice requires your approval before payment can be processed.\\n\"\n    \"Vendor: Great Lakes Furniture & Design Co. (V-00009).\\n\"\n    \"Invoice: GLFD-0441, dated 2026-03-20\\n\"\n    \"Amount: $7,900.00.\\n\"\n    \"Description: Executive conference table, 10-person, walnut finish.\\n\"\n    \"GL Code: 1000-100-6200.\\n\"\n    \"Please reply APPROVED or REJECTED with reason.\\n\"\n    \"If no response is received within 5 business days, the invoice will be placed on APPR hold and the vendor will be notified of the delay.\"\n)\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n    try:\n        with open(mailbox_path, 'r') as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n    emails = data.get('emails', [])\n    drafts = [e for e in emails if str(e.get('folder', '')).lower() == 'drafts']\n\n    if not drafts:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No emails found in Drafts folder\"}\n\n    # Pick best matching draft by subject keywords\n    best_draft = drafts[0]\n    best_score = -1\n    for draft in drafts:\n        subject = str(draft.get('subject', '')).lower()\n        score = ('glfd-0441' in subject) + ('great lakes' in subject)\n        if score > best_score:\n            best_score = score\n            best_draft = draft\n\n    draft = best_draft\n\n    to_raw = draft.get('to_addr', '') or draft.get('to', '') or ''\n    if isinstance(to_raw, list):\n        to_addr = ' '.join([str(x).lower().strip() for x in to_raw])\n    else:\n        to_addr = str(to_raw).lower()\n\n    cc_raw = draft.get('cc_addr', '') or draft.get('cc', '') or ''\n    if isinstance(cc_raw, list):\n        cc_addr = ' '.join([str(x).lower().strip() for x in cc_raw])\n    else:\n        cc_addr = str(cc_raw).lower()\n\n    subject = str(draft.get('subject', ''))\n    body = str(draft.get('body_text', '') or draft.get('body', '') or '')\n\n    checks = {\n        'to: m.chen@meridianpartners.com': 'm.chen@meridianpartners.com' in to_addr,\n        'to: r.whitfield@meridianpartners.com': 'r.whitfield@meridianpartners.com' in to_addr,\n        'cc: ap@meridianpartners.com': 'ap@meridianpartners.com' in cc_addr,\n        'subject exact match': normalize(subject) == normalize(EXPECTED_SUBJECT),\n        'body exact match': normalize(body) == normalize(EXPECTED_BODY),\n    }\n\n    failed = [k for k, v in checks.items() if not v]\n    passed = [k for k, v in checks.items() if v]\n    score = len(passed) / len(checks)\n\n    if not failed:\n        return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Draft email verified: all {len(checks)} checks passed.\"}\n\n    details = f\"Failed: {', '.join(failed)}. Passed: {', '.join(passed)}.\"\n    if 'subject exact match' in failed:\n        details += f\" Subject found: '{subject}'.\"\n    if 'body exact match' in failed:\n        norm_actual = normalize(body)\n        norm_expected = normalize(EXPECTED_BODY)\n        if norm_actual == norm_expected:\n            details += \" Body: normalized content matches — difference is line-break/formatting only.\"\n        else:\n            actual_words = norm_actual.split()\n            expected_words = norm_expected.split()\n            divergence = None\n            for i, (aw, ew) in enumerate(zip(actual_words, expected_words)):\n                if aw != ew:\n                    start = max(0, i - 3)\n                    expected_ctx = ' '.join(expected_words[start:i+6])\n                    actual_ctx = ' '.join(actual_words[start:i+6])\n                    divergence = f\"First divergence at word {i+1}: expected '...{expected_ctx}...' | got '...{actual_ctx}...'\"\n                    break\n            if divergence is None:\n                divergence = f\"Word count differs: expected {len(expected_words)}, got {len(actual_words)}\"\n            details += f\" Body diff: {divergence}\"\n\n\n    return {\"pass\": False, \"score\": score, \"feedback\": details}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "e64b48a4-131c-4e84-b578-61ab0558ed20",
    "sort_order": 2,
    "rubric_text": "In slack_data.json, no new Slack messages must have been sent (the agent must not have sent any Slack messages as part of this task)",
    "verifier_code": "from pathlib import Path\nimport json\nimport time\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check Slack data\"}\n\n    slack_path = Path(external_services_path) / 'slack_data.json'\n    if not slack_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"slack_data.json not found at {slack_path}\"}\n\n    try:\n        with open(slack_path, 'r') as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse slack_data.json: {e}\"}\n\n    messages = data.get('messages', {})\n\n    # Look for any messages that mention Great Lakes, GLFD-0441, or 7900 that might have been sent by the agent\n    # The task says 'No new Slack messages are sent'\n    # We check if there are any messages related to this invoice task\n    task_related_messages = []\n    for channel_id, msgs in messages.items():\n        if not isinstance(msgs, list):\n            continue\n        for msg in msgs:\n            if not isinstance(msg, dict):\n                continue\n            text = str(msg.get('text', '')).lower()\n            if (\n                'great lakes' in text\n                or 'glfd-0441' in text\n                or 'glfd0441' in text\n                or ('7,900' in text and 'furniture' in text)\n                or ('7900' in text and 'furniture' in text)\n            ):\n                task_related_messages.append({'channel': channel_id, 'text': msg.get('text', '')[:100]})\n\n    if task_related_messages:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Found {len(task_related_messages)} Slack message(s) related to Great Lakes/GLFD-0441 that should not have been sent: {task_related_messages}\"}\n\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No Slack messages related to Great Lakes Furniture or GLFD-0441 found, as expected\"}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_1774390462114",
    "sort_order": 3,
    "rubric_text": "No fields are populated after the \"Status\" column in Row 13 of `ap_ledger.xlsx`.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n    if not xlsx_files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n    ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n    target_files = ledger_files if ledger_files else xlsx_files\n\n    for fpath in target_files:\n        try:\n            wb = openpyxl.load_workbook(fpath, data_only=True)\n            ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n            status_col = None\n            for col in range(1, ws.max_column + 1):\n                hdr = ws.cell(row=1, column=col).value\n                if hdr is not None and str(hdr).strip().lower() == \"status\":\n                    status_col = col\n                    break\n\n            if status_col is None:\n                return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find 'Status' column header in {fpath.name}\"}\n\n            populated_after_status = []\n            for col in range(status_col + 1, ws.max_column + 1):\n                cell_value = ws.cell(row=13, column=col).value\n                if cell_value is None:\n                    continue\n                if cell_value == 0 or cell_value == 0.0:\n                    continue\n                if str(cell_value).strip() == \"\" or str(cell_value).strip() == \"0\":\n                    continue\n                col_header = ws.cell(row=1, column=col).value\n                populated_after_status.append((col, col_header, cell_value))\n\n            if not populated_after_status:\n                return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 13 has no populated fields after 'Status' (column {status_col})\"}\n            else:\n                details = \"; \".join([f\"Col {c} ('{h}'): '{v}'\" for c, h, v in populated_after_status])\n                return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 13 has {len(populated_after_status)} populated field(s) after 'Status': {details}\"}\n\n        except Exception as e:\n            continue\n\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read ap_ledger.xlsx\"}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1774486274859",
    "sort_order": 4,
    "rubric_text": "In `mailbox.json`,no more than one email must exist in the Drafts folder, and no emails related to Great Lakes Furniture & Design or invoice GLFD-0441 must exist in the Sent folder.",
    "verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox\"}\n\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"mailbox.json not found at {mailbox_path}\"}\n\n    try:\n        with open(mailbox_path, 'r') as f:\n            data = json.load(f)\n    except Exception as e:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to parse mailbox.json: {e}\"}\n\n    emails = data.get('emails', [])\n    drafts = [e for e in emails if str(e.get('folder', '')).lower() == 'drafts']\n    sent = [e for e in emails if str(e.get('folder', '')).lower() == 'sent']\n\n    failures = []\n\n    if len(drafts) != 1:\n        failures.append(f\"Expected exactly 1 draft, found {len(drafts)}\")\n\n    sent_related = [\n        e for e in sent\n        if any(kw in str(e.get('subject', '')).lower() or kw in str(e.get('body_text', '')).lower() or kw in str(e.get('body', '')).lower() or kw in str(e.get('to_addr', '')).lower()\n               for kw in ['great lakes', 'glfd-0441', 'orders@greatlakesfurniture.com'])\n    ]\n\n    if sent_related:\n        failures.append(f\"Found {len(sent_related)} sent email(s) related to Great Lakes/GLFD-0441\")\n\n    if failures:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(failures)}\n\n    return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Exactly 1 draft found and no related emails in Sent folder\"}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1774487311749",
    "sort_order": 5,
    "rubric_text": "In `ap_ledger.xlsx`, the Invoice Register must contain no more than 13 rows of data (header row plus 12 invoice entries, including the new GLFD-0441 entry in row 13).",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n    if not xlsx_files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n    ledger_files = [f for f in xlsx_files if 'ledger' in f.name.lower() or 'ap' in f.name.lower()]\n    target_files = ledger_files if ledger_files else xlsx_files\n\n    for fpath in target_files:\n        try:\n            wb = openpyxl.load_workbook(fpath, data_only=True)\n            ws = wb[\"Invoice Register\"] if \"Invoice Register\" in wb.sheetnames else wb.worksheets[0]\n\n            extra_rows = []\n            for row in range(14, ws.max_row + 1):\n                if any(ws.cell(row=row, column=col).value is not None for col in range(1, ws.max_column + 1)):\n                    extra_rows.append(row)\n\n            if extra_rows:\n                return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Data found beyond row 13 in {fpath.name}: rows {extra_rows}\"}\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Invoice Register contains no data beyond row 13\"}\n        except Exception:\n            continue\n\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read ap_ledger.xlsx\"}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1774487313895",
    "sort_order": 6,
    "rubric_text": "In `vendor_master.xlsx`, no new vendor rows must have been added (the file must contain no more than 12 rows: 1 header row and 11 vendor rows).",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('*.xlsx'))\n    if not xlsx_files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n    vendor_files = [f for f in xlsx_files if 'vendor' in f.name.lower()]\n    if not vendor_files:\n        return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No vendor_master.xlsx found in workspace\"}\n\n    for fpath in vendor_files:\n        try:\n            wb = openpyxl.load_workbook(fpath, data_only=True)\n            ws = wb.worksheets[0]\n\n            extra_rows = []\n            for row in range(13, ws.max_row + 1):\n                if any(ws.cell(row=row, column=col).value is not None for col in range(1, ws.max_column + 1)):\n                    extra_rows.append(row)\n\n            if extra_rows:\n                return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"New vendor rows found in {fpath.name} beyond row 12: rows {extra_rows}\"}\n            return {\"pass\": True, \"score\": 1.0, \"feedback\": \"vendor_master.xlsx contains no extra vendor rows\"}\n        except Exception:\n            continue\n\n    return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Could not find or read vendor_master.xlsx\"}\n",
    "criterion_type": "incorrect_behavior"
  }
]