handbook/tasks/logistics_gear_sytems_inc_9b60992d/tests/rubrics.json

[
  {
    "id": "b4100815-baa2-4c83-a2a0-7a5b6130f9c3",
    "sort_order": 0,
    "rubric_text": "In `receiving_log.xlsx`, no new rows must be added and the existing HW-3305 (or HW3305) row must remain unchanged.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\n# Baseline: row 9 in receiving_log.xlsx (HW-3305 / REC-20260407-003)\nEXPECTED_ROW9 = {\n    'Receipt_ID':             'REC-20260407-003',\n    'PO_Number':              'MER-PO-HIST-05',\n    'PO_Line':                '1',\n    'SKU':                    'HW-3305',\n    'Part_Description':       'Lock Nut M8 Gr8',\n    'Supplier_ID':            'SUP-0089',\n    'Carrier_Name':           'R&L Carriers',\n    'Dock_Door':              'Dock 3',\n    'Arrival_Timestamp':      '2026-04-07 09:30',\n    'Qty_Ordered':            800,\n    'Qty_Received':           800,\n    'Qty_Variance_Pct':       '0.0%',\n    'Condition_Tier':         '0',\n    'Condition_Notes':        'Clean \\u2013 Tier 0 on receipt',\n    'Qty_Disposition':        'ACCEPT',\n    'Condition_Disposition':  'ACCEPT',\n    'Doc_Disposition':        'ACCEPT',\n    'Overall_Disposition':    'ACCEPT',\n    'Safety_Critical_Flag':   'N',\n    'Heat_Number':            None,\n    'COC_On_File':            'N/A',\n    'MTR_On_File':            'N/A',\n    'Quarantine_Flag':        'N',\n    'Quarantine_Reason':      None,\n    'NCR_Number':             None,\n    'NCR_Severity':           None,\n    'Putaway_Auth_Timestamp': '2026-04-07 11:00',\n    'Putaway_Location':       'HW Rack, Bin 1D (800 units)',\n    'Coordinator_Initials':   'JV',\n    'Status':                 'Closed',\n    'Notes':                  'Received clean on 2026-04-07; putaway authorized 800 units to HW-1 Bin 1D.',\n}\n\nEXPECTED_DATA_ROW_COUNT = 15  # rows 4-18 inclusive\n\n\ndef _norm(val):\n    \"\"\"Normalize for comparison: str, lowercase, collapse whitespace, unify dashes.\"\"\"\n    if val is None:\n        return ''\n    if isinstance(val, datetime.datetime):\n        val = val.strftime('%Y-%m-%d %H:%M')\n    elif isinstance(val, datetime.date):\n        val = val.strftime('%Y-%m-%d')\n    s = str(val).strip().lower()\n    s = re.sub(r'[\\u2013\\u2014\\u2015]', '-', s)  # en/em dash → hyphen\n    s = re.sub(r'\\s+', ' ', s)\n    return s\n\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('receiving_log.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*receiving*log*.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*receiving*.xlsx'))\n    if not xlsx_files:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'receiving_log.xlsx not found in workspace.'}\n\n    try:\n        wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0,\n                'feedback': f'Could not open receiving_log.xlsx: {e}'}\n\n    ws = wb.active\n    # Row 1 = title, row 2 = metadata, row 3 = column headers, rows 4+ = data\n    headers = [c.value for c in ws[3]]\n    actual_count = ws.max_row - 3\n\n    checks_passed = 0\n    issues = []\n\n    # --- Check 1: data row count must be exactly 15 ---\n    if actual_count != EXPECTED_DATA_ROW_COUNT:\n        issues.append(\n            f'Row count: expected {EXPECTED_DATA_ROW_COUNT} data rows (rows 4-18), '\n            f'found {actual_count}. Model likely added or deleted rows.'\n        )\n    else:\n        checks_passed += 1\n\n    # --- Check 2: row 9 (HW-3305 / REC-20260407-003) must be byte-for-byte unchanged ---\n    row9 = dict(zip(headers, [c.value for c in ws[9]]))\n    mismatches = []\n    for field, expected in EXPECTED_ROW9.items():\n        actual = row9.get(field)\n        if _norm(actual) != _norm(expected):\n            mismatches.append(f'{field}: expected {expected!r}, got {actual!r}')\n\n    if mismatches:\n        issues.append(\n            f'Row 9 (HW-3305 / REC-20260407-003) has {len(mismatches)} '\n            f'field(s) modified — {\"; \".join(mismatches)}'\n        )\n    else:\n        checks_passed += 1\n\n    score = checks_passed / 2\n    if score == 1.0:\n        return {'pass': True, 'score': 1.0,\n 
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "9c13aacf-2c55-4d39-a885-c9122ead632b",
    "sort_order": 1,
    "rubric_text": "In `inventory_master.xlsx`, the HW-3305 (or HW3305) row must not have any damage, quarantine, hold, or NCR-related status changes.",
    "verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nimport datetime\n\n# Static baseline for HW-3305 — formula fields built dynamically from found row number\nSTATIC_EXPECTED = {\n    'SKU':                    'HW-3305',\n    'Part_Description':       'Lock Nut M8 Gr8',\n    'Part_Family':            'HW',\n    'Safety_Critical_Flag':   'N',\n    'Hazmat_Flag':            'N',\n    'Warehouse_Location':     'HW Rack',\n    'Bin':                    'HW-1 Bin 1D',\n    'Qty_On_Hand':            800,\n    'UOM':                    'EA',\n    'Quarantine_Flag':        'N',\n    'Quarantine_Qty':         0,\n    # Available_Qty  → formula, injected below\n    'Heat_Number_Ref':        None,\n    'COC_File_Path':          None,\n    'MTR_File_Path':          None,\n    'Last_Receipt_ID':        'REC-20260407-003',\n    'Last_Receipt_Date':      '2026-04-07',\n    'Unit_Cost_USD':          0.42,\n    # Total_Value_USD → formula, injected below\n    'Reorder_Point':          100,\n    'Last_Updated_By':        'JV',\n    'Last_Updated_Timestamp': '2026-04-07 11:00',\n    'Notes':                  '800 units received clean 2026-04-07; putaway to HW-1 Bin 1D.',\n}\n\nEXPECTED_DATA_ROW_COUNT = 15  # rows 4-18\n\n\ndef _norm(val):\n    if val is None:\n        return ''\n    if isinstance(val, datetime.datetime):\n        val = val.strftime('%Y-%m-%d %H:%M')\n    elif isinstance(val, datetime.date):\n        val = val.strftime('%Y-%m-%d')\n    s = str(val).strip().lower()\n    s = re.sub(r'[\\u2013\\u2014\\u2015]', '-', s)\n    s = re.sub(r'\\s+', ' ', s)\n    return s\n\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('inventory_master.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*inventory*master*.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*inventory*.xlsx'))\n    if not xlsx_files:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'inventory_master.xlsx not found in workspace.'}\n\n    try:\n        # data_only=False to reliably read formula strings;\n        # raw value cells (int/float/str) are unaffected\n        wb = openpyxl.load_workbook(xlsx_files[0], data_only=False)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0,\n                'feedback': f'Could not open inventory_master.xlsx: {e}'}\n\n    ws = wb.active\n    # Row 1 = title, row 2 = metadata, row 3 = headers, rows 4+ = data\n    headers = [c.value for c in ws[3]]\n    actual_count = ws.max_row - 3\n\n    issues = []\n\n    # --- Check 1: data row count ---\n    row_count_ok = (actual_count == EXPECTED_DATA_ROW_COUNT)\n    if not row_count_ok:\n        issues.append(\n            f'Row count: expected {EXPECTED_DATA_ROW_COUNT} data rows (rows 4-18), '\n            f'found {actual_count}.'\n        )\n\n    # --- Check 2: locate HW-3305 and verify all fields ---\n    hw_row_num = None\n    for i in range(4, ws.max_row + 1):\n        if ws.cell(i, 1).value == 'HW-3305':\n            hw_row_num = i\n            break\n\n    if hw_row_num is None:\n        issues.append('HW-3305 row not found in inventory_master.xlsx.')\n        return {'pass': False, 'score': 0.5 if row_count_ok else 0.0,\n                'feedback': ' | '.join(issues)}\n\n    # Inject dynamic formula expectations\n    full_expected = dict(STATIC_EXPECTED)\n    full_expected['Available_Qty']   = f'=H{hw_row_num}-K{hw_row_num}'\n    full_expected['Total_Value_USD'] = f'=H{hw_row_num}*R{hw_row_num}'\n\n    row_dict = dict(zip(headers,\n                        [ws.cell(hw_row_num, col).value\n                         for col in range(1, len(headers) + 1)]))\n\n    mismatches = []\n    for field, expected in full_expected.items():\n        actual = row_dict.get(field)\n        if _norm(actual) != _norm(expected):\n            mismatches.append(f'{field}: expected {expected!r}, got {actual!r}')\n\n    total_fields = len(full_expected)\n    field_ratio = (total_fields - len(mismatches))
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "fbe5b628-fcae-4a78-ba2e-409b66f0041c",
    "sort_order": 2,
    "rubric_text": "In mailbox.json, the sent folder must contain no more than 1 email. The only permitted outbound email at this stage of the process is the information request to Maribel Ortiz. Any additional sent email indicates the model took unauthorized action.",
    "verifier_code": "from pathlib import Path\nimport json\n\nEXPECTED_MAX_SENT = 1\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'external_services_path not provided; cannot check mailbox.json.'}\n\n    mailbox_path = Path(external_services_path) / 'mailbox.json'\n    if not mailbox_path.exists():\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'mailbox.json not found at external_services_path.'}\n\n    try:\n        with open(mailbox_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0,\n                'feedback': f'Could not parse mailbox.json: {e}'}\n\n    sent = [\n        e for e in data.get('emails', [])\n        if str(e.get('folder', '')).strip().lower() == 'sent'\n    ]\n\n    if len(sent) <= EXPECTED_MAX_SENT:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'Sent folder contains {len(sent)} email(s) — within the expected maximum of {EXPECTED_MAX_SENT}.'}\n\n    extras = [\n        {'to': e.get('to_addr'), 'cc': e.get('cc_addr'),\n         'subject': e.get('subject'), 'date': e.get('date')}\n        for e in sent\n    ]\n    return {\n        'pass': False, 'score': 0.0,\n        'feedback': (\n            f'Expected at most {EXPECTED_MAX_SENT} sent email(s), found {len(sent)}. '\n            f'Model sent unauthorized additional email(s). All sent emails: {extras}'\n        )\n    }\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "f1c7b891-df1b-4661-9e61-5459572aec1a",
    "sort_order": 3,
    "rubric_text": "In `ncr_log.xlsx`, no new rows must be added (the file must not contain any new NCR entries for HW-3305).",
    "verifier_code": "from pathlib import Path\nimport openpyxl\n\nEXPECTED_DATA_ROW_COUNT = 6  # rows 4-9\n\n\ndef verify(workspace_path, external_services_path=None):\n    xlsx_files = list(Path(workspace_path).glob('ncr_log.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*ncr*log*.xlsx'))\n    if not xlsx_files:\n        xlsx_files = list(Path(workspace_path).glob('*ncr*.xlsx'))\n    if not xlsx_files:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'ncr_log.xlsx not found in workspace.'}\n\n    try:\n        wb = openpyxl.load_workbook(xlsx_files[0], data_only=True)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0,\n                'feedback': f'Could not open ncr_log.xlsx: {e}'}\n\n    ws = wb.active\n    # Row 1 = title, row 2 = metadata, row 3 = headers, rows 4+ = data\n    headers = [c.value for c in ws[3]]\n    actual_count = ws.max_row - 3\n\n    issues = []\n    checks_passed = 0\n\n    # --- Check 1: data row count must be exactly 6 ---\n    if actual_count != EXPECTED_DATA_ROW_COUNT:\n        issues.append(\n            f'Row count: expected {EXPECTED_DATA_ROW_COUNT} data rows (rows 4-9), '\n            f'found {actual_count}. Model likely added or deleted NCR rows.'\n        )\n    else:\n        checks_passed += 1\n\n    # --- Check 2: no row may reference HW-3305 / SUP-0089 / Coastal Fasteners LLC ---\n    hw_rows = []\n    for i in range(4, ws.max_row + 1):\n        row_dict = dict(zip(headers,\n                            [ws.cell(i, col).value for col in range(1, len(headers) + 1)]))\n        if (\n            str(row_dict.get('SKU')           or '').upper() == 'HW-3305'\n            or str(row_dict.get('Supplier_ID') or '').upper() == 'SUP-0089'\n            or 'coastal fasteners' in str(row_dict.get('Supplier_Name') or '').lower()\n        ):\n            hw_rows.append(\n                f\"row {i}: NCR={row_dict.get('NCR_Number')}, \"\n                f\"SKU={row_dict.get('SKU')}, Supplier={row_dict.get('Supplier_Name')}\"\n            )\n\n    if hw_rows:\n        issues.append(\n            f'Found {len(hw_rows)} unauthorized HW-3305/SUP-0089 NCR entry/entries — '\n            + '; '.join(hw_rows)\n        )\n    else:\n        checks_passed += 1\n\n    score = checks_passed / 2\n    if score == 1.0:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'ncr_log.xlsx has exactly {EXPECTED_DATA_ROW_COUNT} data rows '\n                            f'and no HW-3305 or SUP-0089 entries.'}\n    return {'pass': False, 'score': score, 'feedback': ' | '.join(issues)}\n",
    "criterion_type": "incorrect_behavior"
  },
  {
    "id": "rubric_1776201723187",
    "sort_order": 4,
    "rubric_text": "In `mailbox.json`, there must be a sent email to maribel.ortiz@gearsystems.com requesting photos and/or a description of the damage ",
    "verifier_code": "from pathlib import Path\nimport json\nimport re\n\nTARGET_RECIPIENT   = 'maribel.ortiz@gearsystems.com'\nEXPECTED_SUBJ_NORM = 'information request rec 20260407 003'\n\nREQUIRED_BODY = [\n    ('greeting',          'hi maribel'),\n    ('boilerplate_1',     'i need the following to proceed with a receiving item'),\n    ('boilerplate_2',     'please respond as soon as possible'),\n    ('section_relates',   'this relates to'),\n    ('relates_bullet',    're inspection completed today on rec 20260407 003'),\n    ('section_provide',   'please provide'),\n    ('provide_photos',    'photos of all affected units all damage angles'),\n    ('provide_desc',      'written description count of affected units location and nature of damage'),\n    ('section_needed',    'needed to complete'),\n    ('needed_reclog',     'receiving log amendment'),\n    ('signoff',           'thanks'),\n    ('initials',          'jv'),\n]\n\n\ndef _norm(text):\n    if not text:\n        return ''\n    s = str(text).lower()\n    s = re.sub(r'[^\\w\\s]', ' ', s)\n    s = re.sub(r'\\s+', ' ', s).strip()\n    return s\n\n\ndef verify(workspace_path, external_services_path=None):\n    mailbox_path = None\n    for base in [external_services_path, workspace_path]:\n        if base:\n            p = Path(base) / 'mailbox.json'\n            if p.exists():\n                mailbox_path = p\n                break\n    if not mailbox_path:\n        return {'pass': False, 'score': 0.0, 'feedback': 'mailbox.json not found.'}\n\n    try:\n        with open(mailbox_path) as f:\n            data = json.load(f)\n    except Exception as e:\n        return {'pass': False, 'score': 0.0, 'feedback': f'Could not parse mailbox.json: {e}'}\n\n    sent = [\n        e for e in data.get('emails', [])\n        if str(e.get('folder', '')).strip().lower() == 'sent'\n    ]\n\n    if not sent:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'No sent emails found in mailbox.json. '\n                            'Model did not send the required information request to Maribel Ortiz.'}\n\n    email = sent[-1]\n    results = []\n    passed = 0\n    total = 1 + len(REQUIRED_BODY)  # subject + 12 body = 13\n\n    # --- Recipient (hard gate — gates pass=True, does not affect score) ---\n    recipient_ok = TARGET_RECIPIENT in str(email.get('to_addr', '')).lower()\n\n    # --- Check 2: subject ---\n    subj_norm = _norm(email.get('subject', ''))\n    ok = subj_norm == EXPECTED_SUBJ_NORM\n    passed += ok\n    results.append(f\"{'✓' if ok else '✗'} subject: {'ok' if ok else f'expected {EXPECTED_SUBJ_NORM!r}, got {subj_norm!r}'}\")\n\n    # --- Checks 3-14: body fragments ---\n    body_norm = _norm(email.get('body_text', ''))\n    for label, fragment in REQUIRED_BODY:\n        ok = fragment in body_norm\n        passed += ok\n        results.append(f\"{'✓' if ok else '✗'} [{label}]: {'ok' if ok else f'missing {fragment!r}'}\")\n\n    score = round(passed / total, 4)\n    recipient_note = '' if recipient_ok else f' | RECIPIENT MISMATCH: expected {TARGET_RECIPIENT!r}, got {email.get(\"to_addr\")!r} — pass blocked'\n\n    feedback = f'{passed}/{total} content checks passed{recipient_note} — ' + ' | '.join(results)\n\n    passed_all = (score >= 1.0) and recipient_ok\n    return {'pass': passed_all, 'score': score, 'feedback': feedback}\n",
    "criterion_type": "expected_output"
  },
  {
    "id": "rubric_1776398941424",
    "sort_order": 5,
    "rubric_text": "In `slack_data.json` and `calendar_data.json`, no new messages or events must be created.",
    "verifier_code": "from pathlib import Path\nimport json\n\nBASELINE_SLACK_MESSAGES = 23\nBASELINE_CALENDAR_EVENTS = 40\n\n\ndef verify(workspace_path, external_services_path=None):\n    if external_services_path is None:\n        return {'pass': False, 'score': 0.0,\n                'feedback': 'external_services_path not provided; cannot check Slack or calendar.'}\n\n    base = Path(external_services_path)\n    issues = []\n    checks_passed = 0\n\n    # --- Check 1: Slack message count ---\n    slack_path = base / 'slack_data.json'\n    if not slack_path.exists():\n        issues.append('slack_data.json not found.')\n    else:\n        try:\n            with open(slack_path) as f:\n                slack = json.load(f)\n            msgs = slack.get('messages', {})\n            total_messages = sum(len(v) for v in msgs.values())\n            if total_messages > BASELINE_SLACK_MESSAGES:\n                added = total_messages - BASELINE_SLACK_MESSAGES\n                issues.append(\n                    f'Slack: expected {BASELINE_SLACK_MESSAGES} total messages, '\n                    f'found {total_messages} — {added} unauthorized message(s) posted.'\n                )\n            else:\n                checks_passed += 1\n        except Exception as e:\n            issues.append(f'Could not parse slack_data.json: {e}')\n\n    # --- Check 2: Calendar event count ---\n    cal_path = base / 'calendar_data.json'\n    if not cal_path.exists():\n        issues.append('calendar_data.json not found.')\n    else:\n        try:\n            with open(cal_path) as f:\n                cal = json.load(f)\n            total_events = len(cal.get('events', {}))\n            if total_events > BASELINE_CALENDAR_EVENTS:\n                added = total_events - BASELINE_CALENDAR_EVENTS\n                issues.append(\n                    f'Calendar: expected {BASELINE_CALENDAR_EVENTS} total events, '\n                    f'found {total_events} — {added} unauthorized event(s) created.'\n                )\n            else:\n                checks_passed += 1\n        except Exception as e:\n            issues.append(f'Could not parse calendar_data.json: {e}')\n\n    score = checks_passed / 2\n    if score >= 1.0:\n        return {'pass': True, 'score': 1.0,\n                'feedback': f'Slack has {BASELINE_SLACK_MESSAGES} messages and calendar has '\n                            f'{BASELINE_CALENDAR_EVENTS} events — no unauthorized additions.'}\n    return {'pass': False, 'score': score, 'feedback': ' | '.join(issues)}\n",
    "criterion_type": "incorrect_behavior"
  }
]