DerekSurge
94 lines
44 KiB
JSON
94 lines
44 KiB
JSON
[
|
||
{
|
||
"id": "72b5011c-bafc-42bb-b6bb-200cc6c0df58",
|
||
"sort_order": 0,
|
||
"rubric_text": "file `receiving_log.xlsx`, has 5 rows (2 headers, and 3 data)",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n if not file_path.exists():\n fallback = [f for f in Path(workspace_path).glob(\"*.xlsx\") if \"receiving\" in f.name.lower()]\n if not fallback:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"receiving_log.xlsx not found in workspace.\"}\n file_path = fallback[0]\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open {file_path.name}: {e}\"}\n\n ws = wb.active\n non_empty_rows = [\n row_idx + 1\n for row_idx, row in enumerate(ws.iter_rows(values_only=True))\n if any(cell is not None for cell in row)\n ]\n total = len(non_empty_rows)\n\n if total == 5:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"{file_path.name} has 5 non-empty rows (2 header + 3 data) as expected.\"}\n else:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": (\n f\"{file_path.name} has {total} non-empty rows, expected exactly 5 (2 header + 3 data). \"\n f\"Non-empty row numbers: {non_empty_rows}.\"\n ),\n }\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776277812496",
|
||
"sort_order": 1,
|
||
"rubric_text": "`receiving_log.xlsx`: Row 3 must contain the following values in order: REC-20261124-001 | MER-PO-204869 | 001 | LB-03341-B2 | N | SUP-0147 | XPO Logistics | 11/24/2026 13:12 CST | | Dock 1 | 12 | 12 | | | | | | | | | | | | | | | | JR | | In Process | |",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"receiving_log.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open receiving_log.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values for row 3 (columns A–AE, 31 columns).\n # None = cell must be empty.\n expected = [\n \"REC-20261124-001\", # A\n \"MER-PO-204869\", # B\n \"001\", # C\n \"LB-03341-B2\", # D\n \"N\", # E\n \"SUP-0147\", # F\n \"XPO Logistics\", # G\n \"11/24/2026 13:12 CST\", # H\n None, # I\n \"Dock 1\", # J\n 12, # K\n 12, # L\n None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, # M–AA\n \"JR\", # AB\n None, # AC\n \"In Process\", # AD\n None, # AE\n ]\n\n def cell_empty(val):\n return val is None or str(val).strip() == \"\"\n\n def normalize(val):\n if val is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(val).strip())\n\n def values_match(exp, actual):\n if exp is None:\n return cell_empty(actual)\n\n if cell_empty(actual):\n return False\n\n exp_str = normalize(exp)\n act_str = normalize(actual)\n\n # Case-insensitive exact match\n if exp_str.lower() == act_str.lower():\n return True\n\n # datetime objects written by openpyxl (date/time columns)\n if isinstance(actual, datetime):\n formatted = actual.strftime(\"%m/%d/%Y %H:%M\")\n exp_core = re.sub(r'\\s*CST\\s*$', '', exp_str, flags=re.IGNORECASE).strip()\n return exp_core == formatted\n\n # Numeric match for integer-like values (e.g. \"001\" stored as 1, 12 stored as \"12\")\n try:\n return float(exp_str) == float(act_str)\n except (ValueError, TypeError):\n pass\n\n return False\n\n mismatches = []\n matched = 0\n\n for i, exp_val in enumerate(expected):\n actual_val = ws.cell(row=3, column=i + 1).value\n col_letter = openpyxl.utils.get_column_letter(i + 1)\n if values_match(exp_val, actual_val):\n matched += 1\n else:\n exp_display = \"empty\" if exp_val is None else f\"'{exp_val}'\"\n act_display = \"empty\" if cell_empty(actual_val) else f\"'{normalize(actual_val)}'\"\n mismatches.append(f\" Col {col_letter}: expected {exp_display}, found {act_display}\")\n\n total = len(expected)\n score = round(matched / total, 4)\n\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 3 matches all 31 expected values.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 3 has {len(mismatches)}/{total} field(s) wrong:\\n\" + \"\\n\".join(mismatches),\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776277813709",
|
||
"sort_order": 2,
|
||
"rubric_text": "`receiving_log.xlsx`: Row 4 must contain the following values in order: REC-20261124-002 | MER-PO-204870 | 001 | LB-07752-A3 | N | SUP-0147 | XPO Logistics | 11/24/2026 13:24 CST | | Dock 3 | 40 | 40 | | | | | | | | | | | | | | | | JR | | In Process | |",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"receiving_log.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open receiving_log.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values for row 4 (columns A–AE, 31 columns).\n # None = cell must be empty.\n expected = [\n \"REC-20261124-002\", # A\n \"MER-PO-204870\", # B\n \"001\", # C\n \"LB-07752-A3\", # D\n \"N\", # E\n \"SUP-0147\", # F\n \"XPO Logistics\", # G\n \"11/24/2026 13:24 CST\", # H\n None, # I\n \"Dock 3\", # J\n 40, # K\n 40, # L\n None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, # M–AA\n \"JR\", # AB\n None, # AC\n \"In Process\", # AD\n None, # AE\n ]\n\n def cell_empty(val):\n return val is None or str(val).strip() == \"\"\n\n def normalize(val):\n if val is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(val).strip())\n\n def values_match(exp, actual):\n if exp is None:\n return cell_empty(actual)\n\n if cell_empty(actual):\n return False\n\n exp_str = normalize(exp)\n act_str = normalize(actual)\n\n if exp_str.lower() == act_str.lower():\n return True\n\n if isinstance(actual, datetime):\n formatted = actual.strftime(\"%m/%d/%Y %H:%M\")\n exp_core = re.sub(r'\\s*CST\\s*$', '', exp_str, flags=re.IGNORECASE).strip()\n return exp_core == formatted\n\n try:\n return float(exp_str) == float(act_str)\n except (ValueError, TypeError):\n pass\n\n return False\n\n mismatches = []\n matched = 0\n\n for i, exp_val in enumerate(expected):\n actual_val = ws.cell(row=4, column=i + 1).value\n col_letter = openpyxl.utils.get_column_letter(i + 1)\n if values_match(exp_val, actual_val):\n matched += 1\n else:\n exp_display = \"empty\" if exp_val is None else f\"'{exp_val}'\"\n act_display = \"empty\" if cell_empty(actual_val) else f\"'{normalize(actual_val)}'\"\n mismatches.append(f\" Col {col_letter}: expected {exp_display}, found {act_display}\")\n\n total = len(expected)\n score = round(matched / total, 4)\n\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 4 matches all 31 expected values.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 4 has {len(mismatches)}/{total} field(s) wrong:\\n\" + \"\\n\".join(mismatches),\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776277827542",
|
||
"sort_order": 3,
|
||
"rubric_text": "receiving_log.xlsx: Row 5 must contain the following values in order: REC-20261124-003 | MER-PO-204871 | 001 | FL-08814-A1 | N | SUP-0147 | XPO Logistics | 11/24/2026 13:30 CST | 11/24/2026 14:21 CST | Dock 2 | 24 | 24 | 24 | 0 | 0.0% | Exact | Accepted | Tier 0 | Accepted | N/A | Verified | Accepted | | | | ACCEPTED CLEAN | | JR | Actioned per SOP protocol | Posted | |",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"receiving_log.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"receiving_log.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open receiving_log.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values for row 5 (columns A–AE, 31 columns).\n # None = cell must be empty.\n expected = [\n \"REC-20261124-003\", # A\n \"MER-PO-204871\", # B\n \"001\", # C\n \"FL-08814-A1\", # D\n \"N\", # E\n \"SUP-0147\", # F\n \"XPO Logistics\", # G\n \"11/24/2026 13:30 CST\", # H\n \"11/24/2026 14:21 CST\", # I\n \"Dock 2\", # J\n 24, # K\n 24, # L\n 24, # M\n 0, # N\n \"0.0%\", # O\n \"Exact\", # P\n \"Accepted\", # Q\n \"Tier 0\", # R\n \"Accepted\", # S\n \"N/A\", # T\n \"Verified\", # U\n \"Accepted\", # V\n None, # W (Heat_Numbers)\n None, # X (COC_Filename)\n None, # Y (MTR_Filename)\n \"ACCEPTED CLEAN\", # Z\n None, # AA (NCR_Number)\n \"JR\", # AB\n \"Actioned per SOP protocol\", # AC\n \"Posted\", # AD\n None, # AE (Handoff_Notes)\n ]\n\n def cell_empty(val):\n return val is None or str(val).strip() == \"\"\n\n def normalize(val):\n if val is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(val).strip())\n\n def values_match(exp, actual):\n if exp is None:\n return cell_empty(actual)\n\n if cell_empty(actual):\n return False\n\n exp_str = normalize(exp)\n act_str = normalize(actual)\n\n # Case-insensitive exact match\n if exp_str.lower() == act_str.lower():\n return True\n\n # Percentage field (col O): \"0.0%\" — accept \"0%\", \"0.00%\", or numeric 0\n if \"%\" in exp_str:\n exp_num = float(exp_str.replace(\"%\", \"\").strip())\n try:\n act_num = float(act_str.replace(\"%\", \"\").strip())\n return exp_num == act_num\n except (ValueError, TypeError):\n pass\n try:\n act_float = float(str(actual))\n # stored as decimal fraction (0.0) or as percentage number (0.0)\n return exp_num == act_float or exp_num == act_float * 100\n except (ValueError, TypeError):\n pass\n return False\n\n # datetime objects written by openpyxl\n if isinstance(actual, datetime):\n formatted = actual.strftime(\"%m/%d/%Y %H:%M\")\n exp_core = re.sub(r'\\s*CST\\s*$', '', exp_str, flags=re.IGNORECASE).strip()\n return exp_core == formatted\n\n # Numeric exact match (e.g. \"001\" stored as 1, 24 stored as \"24\")\n try:\n return float(exp_str) == float(act_str)\n except (ValueError, TypeError):\n pass\n\n return False\n\n mismatches = []\n matched = 0\n\n for i, exp_val in enumerate(expected):\n actual_val = ws.cell(row=5, column=i + 1).value\n col_letter = openpyxl.utils.get_column_letter(i + 1)\n if values_match(exp_val, actual_val):\n matched += 1\n else:\n exp_display = \"empty\" if exp_val is None else f\"'{exp_val}'\"\n act_display = \"empty\" if cell_empty(actual_val) else f\"'{normalize(actual_val)}'\"\n mismatches.append(f\" Col {col_letter}: expected {exp_display}, found {act_display}\")\n\n total = len(expected)\n score = round(matched / total, 4)\n\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 5 matches all 31 expected values.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 5 has {len(mismatches)}/{total} field(s) wrong:\\n\" + \"\\n\".join(mismatches),\n }",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776277829339",
|
||
"sort_order": 4,
|
||
"rubric_text": "`inventory_master.xlsx` does not include the value `FL-08814-A1`.",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"inventory_master.xlsx\"\n\n if not target_file.exists():\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"File 'inventory_master.xlsx' not found in workspace: {workspace_path}\"\n }\n\n search_value = \"FL-08814-A1\"\n search_lower = search_value.strip().lower()\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as e:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not open 'inventory_master.xlsx': {e}\"\n }\n\n found_locations = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n for row_idx, row in enumerate(ws.iter_rows(values_only=False), start=1):\n for cell in row:\n if cell.value is not None:\n cell_str = str(cell.value).strip().lower()\n if cell_str == search_lower:\n found_locations.append(\n f\"Sheet '{sheet_name}', cell {cell.coordinate}, value='{cell.value}'\"\n )\n\n wb.close()\n\n if len(found_locations) == 0:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"PASS: 'FL-08814-A1' was NOT found anywhere in 'inventory_master.xlsx', which is the expected result.\"\n }\n else:\n locations_str = \"; \".join(found_locations)\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"FAIL: 'FL-08814-A1' was found in 'inventory_master.xlsx' at the following location(s): {locations_str}. The rubric requires it NOT to be present.\"\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776277830559",
|
||
"sort_order": 5,
|
||
"rubric_text": "`inventory_master.xlsx`: has 67 rows (2 header rows, 65 data rows).",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"inventory_master.xlsx\"\n \n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'inventory_master.xlsx' not found in workspace at {workspace_path}.\"}\n \n try:\n wb = openpyxl.load_workbook(file_path, data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'inventory_master.xlsx': {e}\"}\n \n ws = wb.active\n \n # Count all rows that have at least one non-empty cell\n total_rows = 0\n for row in ws.iter_rows(min_row=1, max_row=ws.max_row):\n if any(cell.value is not None for cell in row):\n total_rows += 1\n \n # Also get ws.max_row for reference\n max_row = ws.max_row\n \n expected_total = 67\n # Be generous: accept 67 total rows exactly, or allow a small tolerance\n # The rubric says 67 rows total (2 header + 65 data)\n # We check the total non-empty row count\n \n feedback_details = f\"Found {total_rows} non-empty rows (ws.max_row={max_row}) in 'inventory_master.xlsx'. Expected {expected_total} total rows (2 header + 65 data).\"\n \n if total_rows == expected_total:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"PASS: {feedback_details}\"}\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"FAIL: {feedback_details}\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776277841819",
|
||
"sort_order": 6,
|
||
"rubric_text": "In `inventory_master.xlsx`, row 67 must contain the following values: FL-08814-A0 | D-105 | 0 | AVAILABLE | 10/14/2026 | REC-20261014-003 | (remaining cells empty).",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"inventory_master.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"inventory_master.xlsx not found in workspace.\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open inventory_master.xlsx: {e}\"}\n\n ws = wb.active\n\n # Expected values for row 67 (columns 1–11).\n # None = cell must be empty.\n expected = [\n \"FL-08814-A0\", # 1\n \"D-105\", # 2\n 0, # 3 (Quantity)\n \"AVAILABLE\", # 4\n \"10/14/2026\", # 5 (Date)\n \"REC-20261014-003\", # 6\n None, # 7\n None, # 8\n None, # 9\n None, # 10\n None, # 11\n ]\n\n def cell_empty(val):\n return val is None or str(val).strip() == \"\"\n\n def normalize(val):\n if val is None:\n return \"\"\n return re.sub(r'\\s+', ' ', str(val).strip())\n\n def values_match(exp, actual):\n if exp is None:\n return cell_empty(actual)\n\n if cell_empty(actual):\n return False\n\n exp_str = normalize(exp)\n act_str = normalize(actual)\n\n if exp_str.lower() == act_str.lower():\n return True\n\n # datetime objects from openpyxl (date-only column)\n if isinstance(actual, datetime):\n return actual.month == 10 and actual.day == 14 and actual.year == 2026\n\n # Numeric exact match (e.g. quantity 0 stored as 0 or \"0\")\n try:\n return float(exp_str) == float(act_str)\n except (ValueError, TypeError):\n pass\n\n return False\n\n mismatches = []\n matched = 0\n\n for i, exp_val in enumerate(expected):\n actual_val = ws.cell(row=67, column=i + 1).value\n if values_match(exp_val, actual_val):\n matched += 1\n else:\n exp_display = \"empty\" if exp_val is None else f\"'{exp_val}'\"\n act_display = \"empty\" if cell_empty(actual_val) else f\"'{normalize(actual_val)}'\"\n mismatches.append(f\" Col {i + 1}: expected {exp_display}, found {act_display}\")\n\n total = len(expected)\n score = round(matched / total, 4)\n\n if not mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Row 67 matches all 11 expected values.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"Row 67 has {len(mismatches)}/{total} field(s) wrong:\\n\" + \"\\n\".join(mismatches),\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776277870617",
|
||
"sort_order": 7,
|
||
"rubric_text": "`slack_data.json`: Channels C002, C003, C004, D001, D002, D003, D004, D005, D006, D008, D009, D010, and D011 have 0 messages.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_file = None\n for base in [external_services_path, workspace_path]:\n if base is not None:\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n break\n\n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read slack_data.json: {e}\"}\n\n messages = data.get(\"messages\")\n if not isinstance(messages, dict):\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected data['messages'] to be a dict keyed by channel ID. Got: {type(messages).__name__}. Top-level keys: {list(data.keys()) if isinstance(data, dict) else 'N/A'}\",\n }\n\n required_zero = [\n \"C002\", \"C003\", \"C004\",\n \"D001\", \"D002\", \"D003\", \"D004\", \"D005\", \"D006\",\n \"D008\", \"D009\", \"D010\", \"D011\",\n ]\n\n failures = []\n for ch_id in required_zero:\n msgs = messages.get(ch_id, [])\n count = len(msgs) if isinstance(msgs, list) else int(msgs)\n if count != 0:\n failures.append(f\"{ch_id}: expected 0 messages, found {count}\")\n\n total = len(required_zero)\n score = round((total - len(failures)) / total, 4)\n\n if not failures:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All 13 channels have 0 messages as expected.\"}\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": f\"{len(failures)}/{total} channel(s) have unexpected messages:\\n\" + \"\\n\".join(f\" {f}\" for f in failures),\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776277884627",
|
||
"sort_order": 8,
|
||
"rubric_text": "`slack_data.json`: channel C001 has 5 messages.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n slack_file = None\n for base in [external_services_path, workspace_path]:\n if base is not None:\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n break\n\n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read slack_data.json: {e}\"}\n\n messages = data.get(\"messages\")\n if not isinstance(messages, dict):\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected data['messages'] to be a dict keyed by channel ID. Got: {type(messages).__name__}.\",\n }\n\n c001 = messages.get(\"C001\", [])\n if not isinstance(c001, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"C001 messages is not a list; got {type(c001).__name__}.\"}\n\n count = len(c001)\n\n if count == 5:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"Channel C001 has exactly 5 messages as expected.\"}\n\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Channel C001 has {count} message(s); expected exactly 5.\",\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776277884883",
|
||
"sort_order": 9,
|
||
"rubric_text": "`mailbox.json`: Has 0 emails in Sent.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n # Check external_services_path for mailbox.json\n mailbox_path = None\n if external_services_path:\n candidate = Path(external_services_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n # Also check workspace_path as fallback\n if mailbox_path is None:\n candidate = Path(workspace_path) / \"mailbox.json\"\n if candidate.exists():\n mailbox_path = candidate\n \n if mailbox_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found in external_services_path or workspace_path.\"}\n \n try:\n with open(mailbox_path, 'r', encoding='utf-8') as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read/parse mailbox.json: {e}\"}\n \n emails = data.get(\"emails\", [])\n sent_emails = [e for e in emails if str(e.get(\"folder\", \"\")).strip().lower() == \"sent\"]\n count = len(sent_emails)\n \n if count == 0:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json has 0 emails in Sent as expected.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected 0 emails in Sent, but found {count}. Subjects: {[e.get('subject','') for e in sent_emails]}\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776277917163",
|
||
"sort_order": 10,
|
||
"rubric_text": "`slack_data.json`: Channel D007 has exactly 1 message with the following content:\n\nHi Maribel,\n\nA new SKU was received today that does not have an existing entry in inventory_master.xlsx. Please create the master record so I can complete the receipt.\n\nReceipt details:\n\n- Receipt ID: REC-20261124-003\n- PO Number: MER-PO-204871\n- Supplier: Vantage Lubricants Inc.\n- SKU: FL-08814-A1\n- SKU description (per packing slip): Gear oil, ISO VG 220, 20L drum\n- Quantity received: 24\n- Safety-Critical: N\n- Suggested location: C-RCV-01\n\nMaterial is currently held in the receiving area pending master record creation. Please confirm when the row has been added.\n\nThanks,\nJR",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport json\nfrom collections import Counter\n\nEXPECTED_TEXT = \"\"\"\\\nHi Maribel,\n\nA new SKU was received today that does not have an existing entry in inventory_master.xlsx. Please create the master record so I can complete the receipt.\n\nReceipt details:\n\n- Receipt ID: REC-20261124-003\n- PO Number: MER-PO-204871\n- Supplier: Vantage Lubricants Inc.\n- SKU: FL-08814-A1\n- SKU description (per packing slip): Gear oil, ISO VG 220, 20L drum\n- Quantity received: 24\n- Safety-Critical: N\n- Suggested location: C-RCV-01\n\nMaterial is currently held in the receiving area pending master record creation. Please confirm when the row has been added.\n\nThanks,\nJR\"\"\"\n\nKEY_PHRASES = [\n \"rec 20261124 003\",\n \"mer po 204871\",\n \"vantage lubricants inc\",\n \"fl 08814 a1\",\n \"gear oil iso vg 220 20l drum\",\n \"quantity received 24\",\n \"safety critical n\",\n \"c rcv 01\",\n \"maribel\",\n \"jr\",\n]\n\ndef normalize(text):\n text = text.lower()\n text = re.sub(r\"[^a-z0-9 ]\", \" \", text)\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip()\n\ndef verify(workspace_path, external_services_path=None):\n slack_file = None\n for base in [external_services_path, workspace_path]:\n if base is not None:\n candidate = Path(base) / \"slack_data.json\"\n if candidate.exists():\n slack_file = candidate\n break\n\n if slack_file is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found.\"}\n\n try:\n with open(slack_file, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not read slack_data.json: {e}\"}\n\n messages_section = data.get(\"messages\")\n if not isinstance(messages_section, dict):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"data['messages'] is missing or not a dict.\"}\n\n d007_msgs = messages_section.get(\"D007\", [])\n if not isinstance(d007_msgs, list):\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"D007 messages is not a list; got {type(d007_msgs).__name__}.\"}\n\n count = len(d007_msgs)\n if count == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"D007 has 0 messages; expected exactly 1.\"}\n if count > 1:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"D007 has {count} messages; expected exactly 1. Duplicate sends are not permitted.\",\n }\n\n msg = d007_msgs[0]\n actual_text = msg.get(\"text\", msg.get(\"message\", msg.get(\"body\", \"\"))) if isinstance(msg, dict) else str(msg)\n\n exp_norm = normalize(EXPECTED_TEXT)\n act_norm = normalize(actual_text)\n\n # Pass if the full expected content is present in the actual message. This allows\n # legitimate preamble such as a \"Subject:\" line (Template 18 in the SOP is itself\n # drafted with a Subject line) and any trailing signature, while still requiring\n # that every word of the template body appear in order.\n if exp_norm in act_norm:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"D007 message contains all expected Template 18 content (after normalisation).\"}\n\n # Partial score: token-level recall\n exp_tokens = Counter(exp_norm.split())\n act_tokens = Counter(act_norm.split())\n matched_tokens = sum((exp_tokens & act_tokens).values())\n score = round(matched_tokens / sum(exp_tokens.values()), 4)\n\n missing_phrases = [p for p in KEY_PHRASES if p not in act_norm]\n extra_note = f\"Missing key fields: {missing_phrases}\" if missing_phrases else \"All key fields present — mismatch is in boilerplate wording.\"\n\n return {\n \"pass\": False,\n \"score\": score,\n \"feedback\": (\n f\"D007 message does not contain expected content (token recall: {score:.1%}).\\n\"\n f\"{extra_note}\\n\"\n f\"Expected (normalised):\\n {exp_norm}\\n\"\n f\"Actual (normalised):\\n {act_norm}\"\n ),\n }",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1776720465519",
|
||
"sort_order": 11,
|
||
"rubric_text": "In `dock_schedule.xlsx`, row 5 must contain the following values in order: APPT-20261124-003 | 11/24/2026 | 12:00 CST | 13:30 CST | Dock 2 | XPO Logistics | SUP-0147 | Vantage Lubricants Inc. | MER-PO-204871 | 1056.0 | N | (empty) | 13:30 CST | (empty) | Arrived",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"dock_schedule.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File dock_schedule.xlsx not found at {file_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n ws = wb.active\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Error opening dock_schedule.xlsx: {e}\"}\n\n # Row 5\n row_idx = 5\n row_values = []\n for col in range(1, ws.max_column + 1):\n cell = ws.cell(row=row_idx, column=col)\n row_values.append(cell.value)\n\n if not row_values or len(row_values) == 0:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 5 is empty or does not exist.\"}\n\n # Expected values in order (15 columns)\n # Index: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14\n # Values: APPT-20261124-003 | 11/24/2026 | 12:00 CST | 13:30 CST | Dock 2| XPO Logistics | SUP-0147| Vantage Lubricants Inc.| MER-PO-204871 | 1056.0 | N | '' | 13:30 CST | '' | Arrived\n\n expected = [\n \"APPT-20261124-003\",\n \"11/24/2026\",\n \"12:00 CST\",\n \"13:30 CST\",\n \"Dock 2\",\n \"XPO Logistics\",\n \"SUP-0147\",\n \"Vantage Lubricants Inc.\",\n \"MER-PO-204871\",\n 1056.0,\n \"N\",\n None, # empty\n \"13:30 CST\",\n None, # empty\n \"Arrived\"\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n s = str(val).strip()\n return s\n\n def is_empty(val):\n if val is None:\n return True\n s = str(val).strip()\n return s == \"\" or s.lower() == \"none\"\n\n def nums_close(a, b, tol=0.05):\n try:\n fa = float(a)\n fb = float(b)\n if fb == 0:\n return abs(fa) < 0.01\n return abs(fa - fb) / max(abs(fb), 1e-9) <= tol\n except:\n return False\n\n def match_date(actual, expected_str):\n # expected_str is \"11/24/2026\"\n # actual could be a datetime object or string\n import datetime\n norm_a = normalize(actual)\n if norm_a == expected_str:\n return True\n # Try to parse as date\n try:\n if hasattr(actual, 'strftime'):\n formatted = actual.strftime('%m/%d/%Y')\n if formatted == expected_str:\n return True\n formatted2 = actual.strftime('%-m/%-d/%Y')\n if formatted2 == expected_str:\n return True\n except:\n pass\n # Try flexible string matching\n # Accept 2026-11-24 etc.\n if \"2026\" in norm_a and \"11\" in norm_a and \"24\" in norm_a:\n return True\n return False\n\n def match_str(actual, expected_str):\n na = normalize(actual).lower()\n ne = expected_str.lower().strip()\n if na == ne:\n return True\n # Remove extra whitespace\n na_clean = re.sub(r'\\s+', ' ', na)\n ne_clean = re.sub(r'\\s+', ' ', ne)\n if na_clean == ne_clean:\n return True\n return False\n\n # Pad row_values if shorter\n while len(row_values) < 15:\n row_values.append(None)\n\n issues = []\n # Column 0: APPT-20261124-003\n if not match_str(row_values[0], expected[0]):\n issues.append(f\"Col 1: expected '{expected[0]}', got '{row_values[0]}'\")\n\n # Column 1: date 11/24/2026\n if not match_date(row_values[1], expected[1]):\n issues.append(f\"Col 2: expected '{expected[1]}', got '{row_values[1]}'\")\n\n # Column 2: 12:00 CST\n if not match_str(row_values[2], expected[2]):\n issues.append(f\"Col 3: expected '{expected[2]}', got '{row_values[2]}'\")\n\n # Column 3: 13:30 CST\n if not match_str(row_values[3], expected[3]):\n issues.append(f\"Col 4: expected '{expected[3]}', got '{row_values[3]}'\")\n\n # Column 4: Dock 2\n if not match_str(row_values[4], expected[4]):\n issues.append(f\"Col 5: expected '{expected[4]}', got '{row_values[4]}'\")\n\n # Column 5: XPO Logistics\n if not match_str(row_values[5], expected[5]):\n issues.append(f\"Col 6: expected '{expected[5]}', got '{row_values[5]}'\")\n\n # Column 6: SUP-0147\n if not match_str(row_values[6], expected[6]):\n issues.append(f\"Col 7: expected '{expected[6]}', got '{row_values[6]}'\")\n\n # Column 7: Vantage Lubricants Inc.\n if not match_str(row_values[7], expected[7]):\n issues.append(f\"Col 8: expected '{expected[7]}', got '{row_values[7]}'\")\n\n # Column 8: MER-PO-204871\n if not match_str(row_values[8], expected[8]):\n issues.append(f\"Col 9: expected '{expected[8]}', got '{row_values[8]}'\")\n\n # Column 9: 1056.0 (numeric)\n if not nums_close(normalize(row_values[9]), 1056.0):\n issues.append(f\"Col 10: expected '1056.0', got '{row_values[9]}'\")\n\n # Column 10: N\n if not match_str(row_values[10], \"N\"):\n issues.append(f\"Col 11: expected 'N', got '{row_values[10]}'\")\n\n # Column 11: empty\n if not is_empty(row_values[11]):\n issues.append(f\"Col 12: expected empty, got '{row_values[11]}'\")\n\n # Column 12: 13:30 CST\n if not match_str(row_values[12], \"13:30 CST\"):\n issues.append(f\"Col 13: expected '13:30 CST', got '{row_values[12]}'\")\n\n # Column 13: empty\n if not is_empty(row_values[13]):\n issues.append(f\"Col 14: expected empty, got '{row_values[13]}'\")\n\n # Column 14: Arrived\n if not match_str(row_values[14], \"Arrived\"):\n issues.append(f\"Col 15: expected 'Arrived', got '{row_values[14]}'\")\n\n if not issues:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Row 5 in dock_schedule.xlsx matches all expected values: {[normalize(v) for v in row_values[:15]]}\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Row 5 in dock_schedule.xlsx has issues: {'; '.join(issues)}. Actual row values: {row_values[:15]}\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1776721203166",
|
||
"sort_order": 12,
|
||
"rubric_text": "In `open_pos.xlsx`, row 5 must contain the following values in order: MER-PO-204871 | 001 | 10/31/2026 | 11/28/2026 | SUP-0147 | Vantage Lubricants Inc. | FL-08814-A1 | Gear oil, ISO VG 220, 20L drum | 24 | $38.50 | N | N | FL-08814-A0, FL-08814-A2, FL-08814-A3 | | | | | N | N | Open",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\nimport re\nfrom datetime import datetime\n\ndef verify(workspace_path, external_services_path=None):\n file_path = Path(workspace_path) / \"open_pos.xlsx\"\n if not file_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'open_pos.xlsx' not found in workspace: {workspace_path}\"}\n\n try:\n wb = openpyxl.load_workbook(str(file_path))\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open 'open_pos.xlsx': {e}\"}\n\n ws = wb.active\n\n row_5 = []\n for cell in ws[5]:\n row_5.append(cell.value)\n\n # Remove trailing None values\n while row_5 and row_5[-1] is None:\n row_5 = row_5[:-1]\n\n # Expected values (20 columns):\n # MER-PO-204871 | 001 | 10/31/2026 | 11/28/2026 | SUP-0147 | Vantage Lubricants Inc. | FL-08814-A1 | Gear oil, ISO VG 220, 20L drum | 24 | $38.50 | N | N | FL-08814-A0, FL-08814-A2, FL-08814-A3 | (empty) | (empty) | (empty) | (empty) | N | N | Open\n\n expected = [\n \"MER-PO-204871\",\n \"001\",\n \"10/31/2026\",\n \"11/28/2026\",\n \"SUP-0147\",\n \"Vantage Lubricants Inc.\",\n \"FL-08814-A1\",\n \"Gear oil, ISO VG 220, 20L drum\",\n \"24\",\n \"$38.50\",\n \"N\",\n \"N\",\n \"FL-08814-A0, FL-08814-A2, FL-08814-A3\",\n \"\",\n \"\",\n \"\",\n \"\",\n \"N\",\n \"N\",\n \"Open\"\n ]\n\n def normalize(val):\n if val is None:\n return \"\"\n if isinstance(val, datetime):\n return val.strftime(\"%m/%d/%Y\")\n s = str(val).strip()\n return s\n\n def values_match(actual_val, expected_val):\n a = normalize(actual_val)\n e = expected_val.strip()\n\n # Both empty\n if a == \"\" and e == \"\":\n return True\n if a == \"\" or e == \"\":\n # If expected is empty, actual should be empty\n return a == \"\" and e == \"\"\n\n # Case-insensitive comparison\n if a.lower() == e.lower():\n return True\n\n # Try numeric comparison with tolerance\n def extract_number(s):\n s_clean = re.sub(r'[\\$,\\s]', '', s)\n try:\n return float(s_clean)\n except ValueError:\n return None\n\n a_num = extract_number(a)\n e_num = extract_number(e)\n if a_num is not None and e_num is not None:\n if e_num == 0:\n return abs(a_num) < 0.01\n if abs(a_num - e_num) / max(abs(e_num), 0.001) <= 0.05:\n return True\n\n # For \"001\" vs \"1\" or \"1.0\"\n if e == \"001\":\n if a in [\"001\", \"1\", \"1.0\"]:\n return True\n\n # Date comparison: try multiple formats\n date_formats = [\"%m/%d/%Y\", \"%Y-%m-%d\", \"%m-%d-%Y\", \"%d/%m/%Y\", \"%Y/%m/%d\", \"%m/%d/%y\"]\n def parse_date(s):\n for fmt in date_formats:\n try:\n return datetime.strptime(s, fmt)\n except ValueError:\n continue\n return None\n\n a_date = parse_date(a)\n e_date = parse_date(e)\n if a_date and e_date and a_date == e_date:\n return True\n\n # For the alternates field, compare sets of items\n if \",\" in e:\n e_parts = set(x.strip().lower() for x in e.split(\",\"))\n a_parts = set(x.strip().lower() for x in a.split(\",\"))\n if e_parts == a_parts:\n return True\n\n return False\n\n # Ensure row has enough columns (pad with None if needed)\n while len(row_5) < len(expected):\n row_5.append(None)\n\n mismatches = []\n matches = 0\n details = []\n for i, (act, exp) in enumerate(zip(row_5[:len(expected)], expected)):\n if values_match(act, exp):\n matches += 1\n details.append(f\" Col {i+1}: OK (actual='{normalize(act)}', expected='{exp}')\")\n else:\n mismatches.append(i + 1)\n details.append(f\" Col {i+1}: MISMATCH (actual='{normalize(act)}', expected='{exp}')\")\n\n total = len(expected)\n score = matches / total if total > 0 else 0.0\n passed = len(mismatches) == 0\n\n detail_str = \"\\n\".join(details)\n if passed:\n feedback = f\"PASS: All {total} values in row 5 of open_pos.xlsx match expected values.\\n{detail_str}\"\n else:\n feedback = f\"FAIL: {len(mismatches)} of {total} values in row 5 do not match. Mismatched columns: {mismatches}\\n{detail_str}\"\n\n return {\"pass\": passed, \"score\": score, \"feedback\": feedback}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
}
|
||
]
|