DerekSurge
101 lines
45 KiB
JSON
101 lines
45 KiB
JSON
[
|
||
{
|
||
"id": "f75d80aa-8e60-416c-bc22-5117609b365d",
|
||
"sort_order": 0,
|
||
"rubric_text": "In `mailbox.json` (external services), there must be exactly 4 or 5 emails in the Sent folder, 6 emails in INBOX, 0 in Drafts, and 0 in Trash",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef _has_addr(email: dict, addr: str) -> bool:\n \"\"\"Check if addr appears in any recipient field of the email.\"\"\"\n for field in (\"to_addr\", \"cc_addr\", \"bcc_addr\"):\n val = email.get(field) or \"\"\n if addr.lower() in val.lower():\n return True\n return False\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path\"}\n\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n sent = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n inbox = [e for e in emails if e.get(\"folder\", \"\").lower() == \"inbox\"]\n drafts = [e for e in emails if e.get(\"folder\", \"\").lower() == \"drafts\"]\n trash = [e for e in emails if e.get(\"folder\", \"\").lower() == \"trash\"]\n\n issues = []\n\n # --- Fixed folder counts ---\n if len(inbox) != 6:\n issues.append(f\"INBOX has {len(inbox)} emails, expected 6\")\n if len(drafts) != 0:\n issues.append(f\"Drafts has {len(drafts)} emails, expected 0\")\n if len(trash) != 0:\n issues.append(f\"Trash has {len(trash)} emails, expected 0\")\n\n # --- Required sent emails ---\n # 1. Vasquez consent form\n vasquez_consent = [\n e for e in sent\n if \"evasquez79@gmail.com\" in (e.get(\"to_addr\") or \"\").lower()\n and \"consent form required\" in (e.get(\"subject\") or \"\").lower()\n and \"vasquez\" in (e.get(\"subject\") or \"\").lower()\n ]\n if not vasquez_consent:\n issues.append(\n \"No sent email to evasquez79@gmail.com with subject matching \"\n \"'Consent Form Required | Vasquez'\"\n )\n\n # 2. Thompson incomplete referral\n thompson_referral = [\n e for e in sent\n if \"abriggs@buffaloheme.com\" in (e.get(\"to_addr\") or \"\").lower()\n and \"incomplete referral\" in (e.get(\"subject\") or \"\").lower()\n and \"thompson\" in (e.get(\"subject\") or \"\").lower()\n ]\n if not thompson_referral:\n issues.append(\n \"No sent email to abriggs@buffaloheme.com with subject matching \"\n \"'Incomplete Referral | Thompson'\"\n )\n\n # 3. Legal forward — both recipients must appear somewhere across sent emails\n # Acceptable as one email (both in to/cc) or two separate emails\n legal_reached = any(_has_addr(e, \"legal@careig.com\") for e in sent)\n lmurphy_reached = any(_has_addr(e, \"lmurphy@careig.com\") for e in sent)\n if not legal_reached:\n issues.append(\n \"legal@careig.com not found as a recipient in any sent email \"\n \"(email_id 7 forward required per SOP §12.1)\"\n )\n if not lmurphy_reached:\n issues.append(\n \"lmurphy@careig.com not found as a recipient in any sent email \"\n \"(email_id 7 forward required per SOP §12.1)\"\n )\n\n # --- Sent count: 4 (one forward) or 5 (two separate forwards) ---\n if len(sent) not in (4, 5):\n issues.append(f\"Sent folder has {len(sent)} emails, expected 4 or 5\")\n\n if issues:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(issues)}\n\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": (\n f\"Folder counts correct: Sent={len(sent)}, INBOX={len(inbox)}, \"\n f\"Drafts={len(drafts)}, Trash={len(trash)}. \"\n \"All required sent emails present.\"\n ),\n }\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "f83bcce7-d89c-4273-804c-49f60991b6c3",
|
||
"sort_order": 1,
|
||
"rubric_text": "In `mailbox.json`, there must be a sent email to evasquez79@gmail.com with subject containing 'Consent Form Required' and 'Vasquez', from intake@careig.com, with attachment consent_form.pdf, and body containing key phrases about signing the consent form",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef _normalize(text: str) -> str:\n \"\"\"Collapse whitespace variants and normalize dashes for exact-phrase matching.\"\"\"\n text = text.replace(\"\\u2014\", \"-\").replace(\"\\u2013\", \"-\").replace(\"\\u2212\", \"-\")\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip().lower()\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path\"}\n\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n sent = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n target = next(\n (e for e in sent if \"evasquez79@gmail.com\" in (e.get(\"to_addr\") or \"\").lower()),\n None,\n )\n if target is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to evasquez79@gmail.com\"}\n\n checks = {} # key -> (passed: bool, detail: str)\n\n # Subject — exact normalized match\n subject = target.get(\"subject\", \"\")\n checks[\"subject\"] = (\n \"careig - consent form required | vasquez\" in _normalize(subject),\n f\"Subject: '{subject}'\",\n )\n\n # From address\n from_addr = target.get(\"from_addr\", \"\")\n checks[\"from_addr\"] = (\n \"intake@careig.com\" in from_addr.lower(),\n f\"From: '{from_addr}'\",\n )\n\n # Attachment — consent_form.pdf\n filenames = [a.get(\"filename\", \"\") for a in target.get(\"attachments\", [])]\n checks[\"attachment_consent_form\"] = (\n any(\"consent_form.pdf\" in fn.lower() for fn in filenames),\n f\"Attachments: {filenames}\",\n )\n\n # Body phrases — exact strings from the template, normalized\n body_norm = _normalize(target.get(\"body_text\", \"\"))\n body_phrases = {\n \"body_greeting\": \"dear elena,\",\n \"body_thank_you\": \"thank you for speaking with us.\",\n \"body_referral\": \"dr. nair has referred you to careig specialty pharmacy for home ivig infusion therapy.\",\n \"body_consent_form\": \"signed consent for treatment and financial responsibility form\",\n \"body_fax_number\": \"faxing to (888) 555-0147\",\n \"body_oop_costs\": \"your estimated out-of-pocket costs before scheduling\",\n \"body_signature_name\": \"mia hamilton\",\n \"body_signature_title\": \"careig specialty pharmacy, intake\",\n \"body_signature_email\": \"intake@careig.com\",\n }\n for key, phrase in body_phrases.items():\n checks[key] = (phrase in body_norm, f\"Phrase not found: '{phrase}'\")\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n failed = [k for k, (ok, _) in checks.items() if not ok]\n total = len(checks)\n score = len(passed) / total\n\n if failed:\n details = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (_, detail) in checks.items() if k in failed)\n feedback = details\n else:\n feedback = f\"All {total} checks passed for Vasquez consent email.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "cce481bc-d02a-443b-b556-902a25b3101b",
|
||
"sort_order": 2,
|
||
"rubric_text": "In `mailbox.json`, there must be a sent email to abriggs@buffaloheme.com with subject containing 'Incomplete Referral' and 'Thompson', from intake@careig.com, and body mentioning Physician NPI and Marcus Thompson",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef _normalize(text: str) -> str:\n text = text.replace(\"\\u2014\", \"-\").replace(\"\\u2013\", \"-\").replace(\"\\u2212\", \"-\")\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip().lower()\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path\"}\n\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n sent = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n\n target = next(\n (e for e in sent if \"abriggs@buffaloheme.com\" in (e.get(\"to_addr\") or \"\").lower()),\n None,\n )\n if target is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No sent email found to abriggs@buffaloheme.com\"}\n\n checks = {}\n\n # Subject\n subject = target.get(\"subject\", \"\")\n checks[\"subject\"] = (\n \"incomplete referral | thompson\" in _normalize(subject),\n f\"Subject: '{subject}'\",\n )\n\n # From address\n from_addr = target.get(\"from_addr\", \"\")\n checks[\"from_addr\"] = (\n \"intake@careig.com\" in from_addr.lower(),\n f\"From: '{from_addr}'\",\n )\n\n # Body phrases — exact strings from the template, normalized\n body_norm = _normalize(target.get(\"body_text\", \"\"))\n body_phrases = {\n \"body_greeting\": \"dear dr. briggs,\",\n \"body_referral\": \"we received a referral for marcus thompson (dob: 1965-03-22) for home ivig infusion therapy.\",\n \"body_thank_you\": \"thank you for the referral.\",\n \"body_missing_item\": \"physician npi\",\n \"body_reply_request\": \"please reply to this email or contact us at your earliest convenience.\",\n \"body_phone\": \"(888) 555-0147\",\n \"body_signature_name\": \"mia hamilton\",\n \"body_signature_title\": \"careig specialty pharmacy, intake\",\n \"body_signature_email\": \"intake@careig.com\",\n }\n for key, phrase in body_phrases.items():\n checks[key] = (phrase in body_norm, f\"Phrase not found: '{phrase}'\")\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n failed = [k for k, (ok, _) in checks.items() if not ok]\n total = len(checks)\n score = len(passed) / total\n\n if failed:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (_, detail) in checks.items() if k in failed)\n else:\n feedback = f\"All {total} checks passed for Thompson incomplete referral email.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "78caecc4-f2b7-4c1b-a527-68d2d3f3f135",
|
||
"sort_order": 3,
|
||
"rubric_text": "In `mailbox.json`, email_id '7' must be forwarded to both legal@careig.com and lmurphy@careig.com",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef _has_addr(email: dict, addr: str) -> bool:\n for field in (\"to_addr\", \"cc_addr\", \"bcc_addr\"):\n val = email.get(field) or \"\"\n if addr.lower() in val.lower():\n return True\n return False\n\n\ndef _is_forward_of(email: dict, msg_id: str) -> bool:\n \"\"\"True if sent email is a forward: in_reply_to matches msg_id, or subject has Fwd:/FW:.\"\"\"\n in_reply_to = email.get(\"in_reply_to\") or \"\"\n if msg_id and msg_id in in_reply_to:\n return True\n subject = email.get(\"subject\") or \"\"\n return bool(re.search(r\"\\b(fwd?:|fw:)\", subject, re.IGNORECASE))\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path\"}\n\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n\n # email_id 7 must exist — hard prerequisite, can't score forward without it\n email7 = next((e for e in emails if str(e.get(\"email_id\", \"\")) == \"7\"), None)\n if email7 is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"email_id '7' not found in mailbox\"}\n\n msg_id_7 = email7.get(\"message_id\", \"\")\n sent = [e for e in emails if e.get(\"folder\", \"\").lower() == \"sent\"]\n forwards = [e for e in sent if _is_forward_of(e, msg_id_7)]\n\n checks = {}\n\n # 1. At least one sent email is identified as a forward of email_id 7\n checks[\"forward_identified\"] = (\n len(forwards) > 0,\n \"No sent email identified as a forward of email_id 7 \"\n \"(checked: in_reply_to matches message_id, or subject contains Fwd:/FW:)\",\n )\n\n # 2 & 3. Both required recipients appear somewhere in the forward email(s)\n checks[\"legal_reached\"] = (\n any(_has_addr(e, \"legal@careig.com\") for e in forwards),\n \"legal@careig.com not found as recipient (to/cc/bcc) in any forward of email_id 7\",\n )\n checks[\"lmurphy_reached\"] = (\n any(_has_addr(e, \"lmurphy@careig.com\") for e in forwards),\n \"lmurphy@careig.com not found as recipient (to/cc/bcc) in any forward of email_id 7\",\n )\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n failed = [k for k, (ok, _) in checks.items() if not ok]\n total = len(checks)\n score = len(passed) / total\n\n if failed:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (ok, detail) in checks.items() if not ok)\n else:\n feedback = f\"All {total} checks passed: email_id 7 forwarded to legal@careig.com and lmurphy@careig.com.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "4add9ac6-b994-4d05-bc72-18142e59ff2c",
|
||
"sort_order": 4,
|
||
"rubric_text": "In `mailbox.json`, there must be NO sent emails after email_id = 7 to mthompson65@yahoo.com, syellen58@aol.com, rchoi@hudsonvalleyneuro.com, or rwhitfield@whitfieldlaw.com",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef _has_addr(email: dict, addr: str) -> bool:\n for field in (\"to_addr\", \"cc_addr\", \"bcc_addr\", \"to\", \"cc\", \"bcc\"):\n val = email.get(field) or \"\"\n if isinstance(val, list):\n for v in val:\n if addr.lower() in str(v).lower():\n return True\n elif addr.lower() in str(val).lower():\n return True\n return False\n\n\ndef _email_id(email: dict):\n \"\"\"Extract numeric email_id from an email dict, tolerant of str or int.\"\"\"\n raw = email.get(\"email_id\", email.get(\"id\", None))\n if raw is None:\n return None\n try:\n return int(raw)\n except (ValueError, TypeError):\n return None\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found at external_services_path\"}\n\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n\n emails = data.get(\"emails\", [])\n\n # We consider sent emails with email_id > 7\n # An email is \"sent\" if its folder is \"sent\" (case-insensitive) OR if it has no folder but was clearly sent.\n # Being generous: also check for folder=None entries that have to_addr set.\n sent = []\n for e in emails:\n folder = (e.get(\"folder\") or \"\").strip().lower()\n eid = _email_id(e)\n # Only consider emails after email_id 7\n if eid is not None and eid > 7:\n if folder in (\"sent\", \"outbox\", \"\"):\n sent.append(e)\n elif eid is None:\n # If no id, we can't determine ordering; be conservative and include if sent\n if folder in (\"sent\", \"outbox\"):\n sent.append(e)\n\n forbidden = [\n \"mthompson65@yahoo.com\",\n \"syellen58@aol.com\",\n \"rchoi@hudsonvalleyneuro.com\",\n \"rwhitfield@whitfieldlaw.com\",\n ]\n\n checks = {}\n for addr in forbidden:\n hits = [e for e in sent if _has_addr(e, addr)]\n if hits:\n subjects = [e.get(\"subject\", \"(no subject)\") for e in hits]\n ids = [_email_id(e) for e in hits]\n checks[addr] = (False, f\"Found {len(hits)} sent email(s) after email_id=7 to {addr} (ids={ids}, subjects={subjects})\")\n else:\n checks[addr] = (True, f\"No sent email after email_id=7 to {addr}\")\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n failed = [k for k, (ok, _) in checks.items() if not ok]\n total = len(checks)\n score = len(passed) / total if total > 0 else 0.0\n\n if failed:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (ok, detail) in checks.items() if not ok)\n # Also include passing info\n pass_info = \"; \".join(f\"PASS [{k}]: {detail}\" for k, (ok, detail) in checks.items() if ok)\n if pass_info:\n feedback = feedback + \"; \" + pass_info\n else:\n feedback = f\"All {total} checks passed: no sent emails after email_id=7 to any forbidden address. Details: \" + \"; \".join(detail for _, (_, detail) in checks.items())\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "4221ddf4-4b60-4f00-acc4-98a3f06ff4cd",
|
||
"sort_order": 5,
|
||
"rubric_text": "In `slack_data.json`, channel #compliance (id: C008) must contain exactly one message with text '[LEGAL] CIG_2026_1024', and channel #intake-alerts (id: C002) must contain exactly one message with text '[INTAKE HOLD] CIG_2026_1026 — Physician NPI missing.'",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\n\ndef _normalize(text: str) -> str:\n text = text.replace(\"\\u2014\", \"-\").replace(\"\\u2013\", \"-\").replace(\"\\u2212\", \"-\")\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip().lower()\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path\"}\n\n with open(slack_path, \"r\") as f:\n data = json.load(f)\n\n messages_by_channel = data.get(\"messages\", {})\n checks = {}\n\n # 1. #compliance (C008): exactly 1 message with exact text \"[LEGAL] CIG_2026_1024\"\n expected_legal = _normalize(\"[LEGAL] CIG_2026_1024\")\n c008_msgs = messages_by_channel.get(\"C008\", [])\n legal_matches = [m for m in c008_msgs if _normalize(m.get(\"text\", \"\")) == expected_legal]\n if len(legal_matches) == 1:\n checks[\"compliance_legal_msg\"] = (True, \"Found exactly 1 '[LEGAL] CIG_2026_1024' in #compliance (C008)\")\n elif len(legal_matches) == 0:\n checks[\"compliance_legal_msg\"] = (\n False,\n f\"No message matching '[LEGAL] CIG_2026_1024' in #compliance (C008). Found: {[m.get('text','') for m in c008_msgs]}\",\n )\n else:\n checks[\"compliance_legal_msg\"] = (\n False,\n f\"Expected exactly 1, found {len(legal_matches)} matching '[LEGAL] CIG_2026_1024' in #compliance (C008)\",\n )\n\n # 2. #intake-alerts (C002): exactly 1 message with exact text \"[INTAKE HOLD] CIG_2026_1026 — Physician NPI missing.\"\n expected_hold = _normalize(\"[INTAKE HOLD] CIG_2026_1026 — Physician NPI missing.\")\n c002_msgs = messages_by_channel.get(\"C002\", [])\n hold_matches = [m for m in c002_msgs if _normalize(m.get(\"text\", \"\")) == expected_hold]\n if len(hold_matches) == 1:\n checks[\"intake_hold_msg\"] = (True, \"Found exactly 1 '[INTAKE HOLD] CIG_2026_1026 - Physician NPI missing.' in #intake-alerts (C002)\")\n elif len(hold_matches) == 0:\n checks[\"intake_hold_msg\"] = (\n False,\n f\"No message matching '[INTAKE HOLD] CIG_2026_1026 - Physician NPI missing.' in #intake-alerts (C002). Found: {[m.get('text','') for m in c002_msgs]}\",\n )\n else:\n checks[\"intake_hold_msg\"] = (\n False,\n f\"Expected exactly 1, found {len(hold_matches)} matching hold message in #intake-alerts (C002)\",\n )\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n score = len(passed) / len(checks)\n\n if score < 1.0:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (ok, detail) in checks.items() if not ok)\n else:\n feedback = \"All 2 checks passed: correct messages in #compliance and #intake-alerts.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "af6b9a4f-2cb5-4399-9a1f-e8e615a15c76",
|
||
"sort_order": 6,
|
||
"rubric_text": "In intake_CIG_2026_1025.xlsx, cell B36 must be '2026-09-28 08:15 AM — Email' (or equivalent date-time and channel) and cell F37 must be 'Form Sent'",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\n\n\ndef _normalize(text: str) -> str:\n text = text.replace(\"\\u2014\", \"-\").replace(\"\\u2013\", \"-\").replace(\"\\u2212\", \"-\")\n text = text.replace(\"--\", \"-\")\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip().lower()\n\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"intake_CIG_2026_1025.xlsx\"\n if not target_file.exists():\n matches = list(Path(workspace_path).glob(\"*1025*.xlsx\"))\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"intake_CIG_2026_1025.xlsx not found in workspace\"}\n target_file = matches[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {ex}\"}\n\n ws = wb.active\n b36_raw = ws[\"B36\"].value\n f37_raw = ws[\"F37\"].value\n b36_str = _normalize(str(b36_raw).strip()) if b36_raw is not None else \"\"\n f37_str = _normalize(str(f37_raw).strip()) if f37_raw is not None else \"\"\n\n checks = {}\n\n # B36: exact normalized match for \"2026-09-28 08:15 AM — Email\"\n expected_b36 = _normalize(\"2026-09-28 08:15 AM — Email\")\n checks[\"B36_contact_log\"] = (\n b36_str == expected_b36,\n f\"B36='{b36_raw}', expected '2026-09-28 08:15 AM — Email'\",\n )\n\n # F37: exact normalized match for \"Form Sent\"\n expected_f37 = _normalize(\"Form Sent\")\n checks[\"F37_consent_status\"] = (\n f37_str == expected_f37,\n f\"F37='{f37_raw}', expected 'Form Sent'\",\n )\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n score = len(passed) / len(checks)\n\n if score < 1.0:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (ok, detail) in checks.items() if not ok)\n else:\n feedback = f\"All {len(checks)} checks passed: B36='{b36_raw}', F37='{f37_raw}'.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "4fb4cb2b-65e3-4275-8ae7-e2d40a3f705e",
|
||
"sort_order": 7,
|
||
"rubric_text": "In `intake_CIG_2026_1026.xlsx`, cell B6 must be 'CIG_2026_1026', cell F6 must be 'Thompson, Marcus', cells B9/B10/F9/F10/C12/B15/D15/F15/B21/C21/D21/E21/B25/C25/D25/F25 must match specified referral data, and cell C15 must be empty, cells F21/G21/B34:B39/F34/F40 must be empty, and F37 must be 'Pending'",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport datetime\nimport openpyxl\n\n\ndef _normalize(text: str) -> str:\n text = text.replace(\"\\u2014\", \"-\").replace(\"\\u2013\", \"-\").replace(\"\\u2212\", \"-\")\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip().lower()\n\n\ndef _cell_str(ws, ref: str) -> str:\n \"\"\"Return cell value as a normalized string, handling datetime objects.\"\"\"\n v = ws[ref].value\n if v is None:\n return \"\"\n if isinstance(v, (datetime.date, datetime.datetime)):\n return str(v.date() if isinstance(v, datetime.datetime) else v)\n return _normalize(str(v))\n\n\ndef _is_empty(ws, ref: str) -> bool:\n v = ws[ref].value\n return v is None or str(v).strip() == \"\"\n\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"intake_CIG_2026_1026.xlsx\"\n if not target_file.exists():\n matches = [m for m in Path(workspace_path).glob(\"*.xlsx\") if \"1026\" in m.name]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"intake_CIG_2026_1026.xlsx not found in workspace\"}\n target_file = matches[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {ex}\"}\n\n ws = wb.active\n checks = {}\n\n # --- Required values ---\n required = [\n (\"B6\", \"CIG_2026_1026\"),\n (\"F6\", \"Thompson, Marcus\"),\n (\"B9\", \"88 Oakdale Ave\"),\n (\"B10\", \"Buffalo, NY 14216\"),\n (\"F9\", \"(716) 482-9031\"),\n (\"F10\", \"mthompson65@yahoo.com\"),\n (\"C12\", \"1965-03-22\"), # date cell — handled by _cell_str\n (\"B15\", \"Alan Briggs\"),\n (\"D15\", \"abriggs@buffaloheme.com\"),\n (\"F15\", \"D80.1\"),\n (\"B21\", \"Empire BlueCross PPO\"),\n (\"C21\", \"EBC-441987320\"),\n (\"D21\", \"GRP-7815\"),\n (\"E21\", \"(800) 553-9603\"),\n (\"B25\", \"Privigen 10%\"),\n (\"C25\", \"25g\"),\n (\"D25\", \"every 3 weeks\"),\n (\"F25\", \"None\"),\n (\"F37\", \"Pending\"),\n ]\n for ref, expected in required:\n actual = _cell_str(ws, ref)\n checks[ref] = (\n actual == _normalize(expected),\n f\"{ref}='{ws[ref].value}', expected '{expected}'\",\n )\n\n # --- Must be empty ---\n empty_cells = [\"C15\", \"F21\", \"G21\", \"F34\", \"F40\"] + [f\"B{r}\" for r in range(34, 40)]\n for ref in empty_cells:\n checks[f\"{ref}_empty\"] = (\n _is_empty(ws, ref),\n f\"{ref}='{ws[ref].value}', expected empty\",\n )\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n score = len(passed) / len(checks)\n\n if score < 1.0:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (ok, detail) in checks.items() if not ok)\n else:\n feedback = f\"All {len(checks)} checks passed for intake_CIG_2026_1026.xlsx.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "2fed24e7-0f75-402c-85ef-1a5e6d88027c",
|
||
"sort_order": 8,
|
||
"rubric_text": "In `intake_CIG_2026_1024.xlsx`, cell F37 must be 'Form Sent' and cell F40 must be empty",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport openpyxl\n\n\ndef _normalize(text: str) -> str:\n text = text.replace(\"\\u2014\", \"-\").replace(\"\\u2013\", \"-\").replace(\"\\u2212\", \"-\")\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip().lower()\n\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"intake_CIG_2026_1024.xlsx\"\n if not target_file.exists():\n matches = [m for m in Path(workspace_path).glob(\"*.xlsx\") if \"1024\" in m.name]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"intake_CIG_2026_1024.xlsx not found in workspace\"}\n target_file = matches[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {ex}\"}\n\n ws = wb.active\n checks = {}\n\n # F37: exact normalized match for \"Form Sent\"\n f37_raw = ws[\"F37\"].value\n f37_str = _normalize(str(f37_raw)) if f37_raw is not None else \"\"\n checks[\"F37_consent_status\"] = (\n f37_str == _normalize(\"Form Sent\"),\n f\"F37='{f37_raw}', expected 'Form Sent'\",\n )\n\n # F40: must be empty\n f40_raw = ws[\"F40\"].value\n checks[\"F40_empty\"] = (\n f40_raw is None or str(f40_raw).strip() == \"\",\n f\"F40='{f40_raw}', expected empty\",\n )\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n score = len(passed) / len(checks)\n\n if score < 1.0:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (ok, detail) in checks.items() if not ok)\n else:\n feedback = f\"All {len(checks)} checks passed for intake_CIG_2026_1024.xlsx.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "fa4e4263-e72f-41a0-ba1e-70c2c3fa038d",
|
||
"sort_order": 9,
|
||
"rubric_text": "In audit_log.xlsx, there must be a row with Date_Time='2026-09-28 08:15 AM', CASE_ID='CIG_2026_1024', Staff_User='Mia Hamilton', Result='STOP'",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport datetime\nimport openpyxl\n\n\ndef _normalize(text: str) -> str:\n text = text.replace(\"\\u2014\", \"-\").replace(\"\\u2013\", \"-\").replace(\"\\u2212\", \"-\")\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip().lower()\n\n\ndef _val_str(v) -> str:\n if v is None:\n return \"\"\n if isinstance(v, datetime.datetime):\n return _normalize(v.strftime(\"%Y-%m-%d %I:%M %p\"))\n if isinstance(v, datetime.date):\n return _normalize(str(v))\n return _normalize(str(v))\n\n\ndef _find_header_row(ws, required: set, max_scan: int = 10):\n \"\"\"Return (header_row_index_1based, col_map) for the first row containing all required columns.\"\"\"\n for i, row in enumerate(ws.iter_rows(min_row=1, max_row=max_scan, values_only=True), 1):\n col = {}\n for j, cell in enumerate(row):\n if cell is not None:\n col[_normalize(str(cell)).replace(\" \", \"_\")] = j # 0-based\n if required.issubset(col.keys()):\n return i, col\n return None, {}\n\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"audit_log.xlsx\"\n if not target_file.exists():\n matches = [m for m in Path(workspace_path).glob(\"*.xlsx\") if \"audit\" in m.name.lower()]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"audit_log.xlsx not found in workspace\"}\n target_file = matches[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {ex}\"}\n\n ws = wb.active\n required_cols = {\"date_time\", \"case_id\", \"staff_user\", \"result\"}\n header_row, col = _find_header_row(ws, required_cols)\n\n if header_row is None:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not find header row with columns {required_cols} in first 10 rows of audit_log.xlsx\",\n }\n\n expected = {\n \"date_time\": _normalize(\"2026-09-28 08:15 AM\"),\n \"case_id\": _normalize(\"CIG_2026_1024\"),\n \"staff_user\": _normalize(\"Mia Hamilton\"),\n \"result\": _normalize(\"STOP\"),\n }\n\n # Find best-matching row for CIG_2026_1024\n best_row = None\n best_hits = -1\n\n for row in ws.iter_rows(min_row=header_row + 1, values_only=True):\n def get(key, r=row):\n idx = col[key]\n return _val_str(r[idx] if idx < len(r) else None)\n\n if get(\"case_id\") != expected[\"case_id\"]:\n continue\n\n candidate = {k: get(k) for k in expected}\n hits = sum(candidate[k] == expected[k] for k in expected)\n if hits > best_hits:\n best_hits = hits\n best_row = candidate\n\n if best_row is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No row with CASE_ID='CIG_2026_1024' found in audit_log.xlsx\"}\n\n checks = {}\n for field, exp_val in expected.items():\n actual = best_row[field]\n checks[field] = (\n actual == exp_val,\n f\"{field}='{actual}', expected '{exp_val}'\",\n )\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n score = len(passed) / len(checks)\n\n if score < 1.0:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (ok, detail) in checks.items() if not ok)\n else:\n feedback = f\"All {len(checks)} checks passed: STOP row for CIG_2026_1024 verified.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1775156790646",
|
||
"sort_order": 10,
|
||
"rubric_text": "In `slack_data.json`, there must be exactly 2 total Slack messages.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {ex}\"}\n\n messages_by_channel = data.get(\"messages\", {})\n total = sum(len(msgs) for msgs in messages_by_channel.values())\n\n if total == 2:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Exactly 2 total Slack messages found across all channels.\"}\n else:\n counts = {ch: len(msgs) for ch, msgs in messages_by_channel.items() if msgs}\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected exactly 2 total Slack messages, found {total}. Non-empty channels: {counts}\",\n }\n",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "rubric_1775156799429",
|
||
"sort_order": 11,
|
||
"rubric_text": "In `slack_data.json`, there must be no messages containing 'BVS ASSIGNED'.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check slack_data.json\"}\n\n slack_path = Path(external_services_path) / \"slack_data.json\"\n if not slack_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"slack_data.json not found at external_services_path\"}\n\n try:\n with open(slack_path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to read slack_data.json: {ex}\"}\n\n messages_by_channel = data.get(\"messages\", {})\n all_messages = [m for msgs in messages_by_channel.values() for m in msgs]\n\n hits = [m for m in all_messages if \"bvs assigned\" in m.get(\"text\", \"\").lower()]\n\n if not hits:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"No messages containing 'BVS ASSIGNED' found in slack_data.json.\"}\n\n examples = [m.get(\"text\", \"\")[:120] for m in hits[:3]]\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Found {len(hits)} message(s) containing 'BVS ASSIGNED': {examples}\",\n }\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1775156903911",
|
||
"sort_order": 12,
|
||
"rubric_text": "In audit_log.xlsx, there must be NO rows with Date_Time='2026-09-28 08:15 AM' and CASE_ID='CIG_2026_1025', NO rows with Date_Time='2026-09-28 08:15 AM' and CASE_ID='CIG_2026_1026', and NO rows with Date_Time='2026-09-28 08:15 AM', CASE_ID='CIG_2026_1024', and Action='BVS Assigned'.",
|
||
"verifier_code": "from pathlib import Path\nimport re\nimport datetime\nimport openpyxl\n\n\ndef _normalize(text: str) -> str:\n text = text.replace(\"\\u2014\", \"-\").replace(\"\\u2013\", \"-\").replace(\"\\u2212\", \"-\")\n text = re.sub(r\"\\s+\", \" \", text)\n return text.strip().lower()\n\n\ndef _val_str(v) -> str:\n if v is None:\n return \"\"\n if isinstance(v, datetime.datetime):\n return _normalize(v.strftime(\"%Y-%m-%d %I:%M %p\"))\n if isinstance(v, datetime.date):\n return _normalize(str(v))\n return _normalize(str(v))\n\n\ndef _find_header_row(ws, required: set, max_scan: int = 10):\n for i, row in enumerate(ws.iter_rows(min_row=1, max_row=max_scan, values_only=True), 1):\n col = {}\n for j, cell in enumerate(row):\n if cell is not None:\n col[_normalize(str(cell)).replace(\" \", \"_\")] = j\n if required.issubset(col.keys()):\n return i, col\n return None, {}\n\n\ndef verify(workspace_path, external_services_path=None):\n target_file = Path(workspace_path) / \"audit_log.xlsx\"\n if not target_file.exists():\n matches = [m for m in Path(workspace_path).glob(\"*.xlsx\") if \"audit\" in m.name.lower()]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"audit_log.xlsx not found in workspace\"}\n target_file = matches[0]\n\n try:\n wb = openpyxl.load_workbook(target_file, data_only=True)\n except Exception as ex:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Failed to open workbook: {ex}\"}\n\n ws = wb.active\n required_cols = {\"date_time\", \"case_id\", \"action\"}\n header_row, col = _find_header_row(ws, required_cols)\n\n if header_row is None:\n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Could not find header row with columns {required_cols} in first 10 rows of audit_log.xlsx\",\n }\n\n target_dt = _normalize(\"2026-09-28 08:15 AM\")\n\n flagged = {\"CIG_2026_1025\": [], \"CIG_2026_1026\": [], \"CIG_2026_1024_bvs\": []}\n\n for row in ws.iter_rows(min_row=header_row + 1, values_only=True):\n def get(key, r=row):\n idx = col.get(key)\n return _val_str(r[idx] if idx is not None and idx < len(r) else None)\n\n if get(\"date_time\") != target_dt:\n continue\n\n case = get(\"case_id\").upper()\n\n if case == \"CIG_2026_1025\":\n flagged[\"CIG_2026_1025\"].append(dict(row=row))\n if case == \"CIG_2026_1026\":\n flagged[\"CIG_2026_1026\"].append(dict(row=row))\n if case == \"CIG_2026_1024\" and \"bvs assigned\" in get(\"action\"):\n flagged[\"CIG_2026_1024_bvs\"].append(dict(row=row))\n\n checks = {\n \"no_1025_at_0815\": (\n len(flagged[\"CIG_2026_1025\"]) == 0,\n f\"Found {len(flagged['CIG_2026_1025'])} row(s) with Date_Time='2026-09-28 08:15 AM' and CASE_ID='CIG_2026_1025'\",\n ),\n \"no_1026_at_0815\": (\n len(flagged[\"CIG_2026_1026\"]) == 0,\n f\"Found {len(flagged['CIG_2026_1026'])} row(s) with Date_Time='2026-09-28 08:15 AM' and CASE_ID='CIG_2026_1026'\",\n ),\n \"no_1024_bvs_at_0815\": (\n len(flagged[\"CIG_2026_1024_bvs\"]) == 0,\n f\"Found {len(flagged['CIG_2026_1024_bvs'])} row(s) with Date_Time='2026-09-28 08:15 AM', CASE_ID='CIG_2026_1024', Action='BVS Assigned'\",\n ),\n }\n\n passed = [k for k, (ok, _) in checks.items() if ok]\n score = len(passed) / len(checks)\n\n if score < 1.0:\n feedback = \"; \".join(f\"FAIL [{k}]: {detail}\" for k, (ok, detail) in checks.items() if not ok)\n else:\n feedback = \"All 3 checks passed: no forbidden audit log rows at 2026-09-28 08:15 AM.\"\n\n return {\"pass\": score == 1.0, \"score\": round(score, 4), \"feedback\": feedback}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "rubric_1775161530767",
|
||
"sort_order": 13,
|
||
"rubric_text": "The file `audit_log.xlsx` has exactly 3 data rows (starting row 6)",
|
||
"verifier_code": "from pathlib import Path\nimport openpyxl\n\ndef verify(workspace_path, external_services_path=None):\n # Look for audit_log.xlsx in workspace\n target_file = Path(workspace_path) / \"audit_log.xlsx\"\n if not target_file.exists():\n # Try glob for any xlsx that might be the audit log\n candidates = list(Path(workspace_path).glob(\"*audit_log*.xlsx\"))\n if not candidates:\n candidates = list(Path(workspace_path).glob(\"*.xlsx\"))\n if not candidates:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"File 'audit_log.xlsx' not found in workspace: {workspace_path}. No xlsx files found.\"}\n # Use first candidate\n target_file = candidates[0]\n\n try:\n wb = openpyxl.load_workbook(str(target_file), data_only=True)\n except Exception as e:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not open '{target_file.name}': {e}\"}\n\n # The rubric says: exactly 3 data rows starting at row 6\n # This means rows 1-5 are header/metadata, and data rows are 6, 7, 8\n # We check that row 6, 7, 8 have data and row 9 does not (or doesn't exist)\n\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n \n # Count data rows starting from row 6\n # A \"data row\" is one that has at least one non-empty cell\n data_row_start = 6\n data_row_count = 0\n max_row_to_check = ws.max_row if ws.max_row else 20\n \n for row_num in range(data_row_start, max_row_to_check + 1):\n row_cells = [ws.cell(row=row_num, column=col).value for col in range(1, (ws.max_column or 10) + 1)]\n has_data = any(cell is not None and str(cell).strip() != '' for cell in row_cells)\n if has_data:\n data_row_count += 1\n # Don't break on empty row in case there are gaps, but typically data is contiguous\n \n if data_row_count == 3:\n return {\n \"pass\": True,\n \"score\": 1.0,\n \"feedback\": f\"Found exactly 3 data rows starting from row 6 in sheet '{sheet_name}' of '{target_file.name}'. Data rows span rows {data_row_start} to {data_row_start + 2}. Total data rows counted: {data_row_count}.\"\n }\n else:\n # Check if another sheet might match\n continue\n\n # If we get here, no sheet matched\n # Provide detailed feedback about what we found\n feedback_parts = []\n for sheet_name in wb.sheetnames:\n ws = wb[sheet_name]\n data_row_start = 6\n data_row_count = 0\n max_row_to_check = ws.max_row if ws.max_row else 20\n \n for row_num in range(data_row_start, max_row_to_check + 1):\n row_cells = [ws.cell(row=row_num, column=col).value for col in range(1, (ws.max_column or 10) + 1)]\n has_data = any(cell is not None and str(cell).strip() != '' for cell in row_cells)\n if has_data:\n data_row_count += 1\n \n feedback_parts.append(f\"Sheet '{sheet_name}': found {data_row_count} data rows starting from row 6 (max_row={ws.max_row})\")\n \n return {\n \"pass\": False,\n \"score\": 0.0,\n \"feedback\": f\"Expected exactly 3 data rows starting from row 6 in '{target_file.name}'. \" + \"; \".join(feedback_parts)\n }\n",
|
||
"criterion_type": "expected_output"
|
||
}
|
||
]
|