DerekSurge
59 lines
23 KiB
JSON
59 lines
23 KiB
JSON
[
|
||
{
|
||
"id": "7e23756f-b4ae-4663-a74d-790bd570af74",
|
||
"sort_order": 0,
|
||
"rubric_text": "In `calendar_data.json` (external services), there must be exactly 286 events in the events object.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found at external_services_path\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n events = data.get(\"events\", {})\n count = len(events)\n if count == 286:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"calendar_data.json contains exactly 286 events as required.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 286 events in calendar_data.json, but found {count}.\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "03437a49-686c-493e-8f2a-4af41397970e",
|
||
"sort_order": 1,
|
||
"rubric_text": "In `calendar_data.json` (external services), there must be an event with summary 'Exit Interview – James Smith' (or close variant) starting at 3:30 PM and ending at 4:30 PM on February 12th, 2026.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n\n events = data.get(\"events\", {})\n if isinstance(events, dict):\n iterator = list(events.items())\n elif isinstance(events, list):\n iterator = [(ev.get(\"id\", f\"idx-{i}\"), ev) for i, ev in enumerate(events)]\n else:\n iterator = []\n\n def _dt(v):\n if isinstance(v, dict):\n return v.get(\"dateTime\") or v.get(\"date\") or \"\"\n return v or \"\"\n\n target_summary_pattern = re.compile(r'exit\\s+interview.*james\\s+smith', re.IGNORECASE)\n matches = [(eid, ev) for eid, ev in iterator if target_summary_pattern.search(ev.get(\"summary\", \"\"))]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No event matching 'Exit Interview - James Smith' found in calendar_data.json\"}\n\n last_feedback = \"No matching event had the expected time.\"\n for eid, ev in matches:\n start_str = _dt(ev.get(\"start\", \"\"))\n end_str = _dt(ev.get(\"end\", \"\"))\n parsed = False\n for fmt in [\"%Y-%m-%dT%H:%M:%S\", \"%Y-%m-%dT%H:%M\", \"%Y-%m-%d %H:%M:%S\", \"%Y-%m-%d %H:%M\"]:\n try:\n start_dt = datetime.strptime(start_str[:19], fmt)\n end_dt = datetime.strptime(end_str[:19], fmt)\n parsed = True\n if (start_dt.year == 2026 and start_dt.month == 2 and start_dt.day == 12\n and start_dt.hour == 15 and start_dt.minute == 30\n and end_dt.year == 2026 and end_dt.month == 2 and end_dt.day == 12\n and end_dt.hour == 16 and end_dt.minute == 30):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found 'Exit Interview - James Smith' event on 2026-02-12 15:30-16:30. Event ID: {eid}\"}\n last_feedback = f\"Found event matching 'Exit Interview - James Smith' but wrong time: start={start_str}, end={end_str}. Expected 2026-02-12 15:30-16:30.\"\n break\n except ValueError:\n continue\n if not parsed:\n last_feedback = f\"Found matching event but could not parse start/end times: start={start_str}, end={end_str}\"\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": last_feedback}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "96456488-26d3-46ed-adda-a47ce5e4da2c",
|
||
"sort_order": 2,
|
||
"rubric_text": "In `calendar_data.json` (external services), there must be an event with summary 'Exit Interview – Teresa Romero' (or close variant) starting at 9:00 AM and ending at 9:30 AM on February 13th, 2026.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n\n events = data.get(\"events\", {})\n if isinstance(events, dict):\n iterator = list(events.items())\n elif isinstance(events, list):\n iterator = [(ev.get(\"id\", f\"idx-{i}\"), ev) for i, ev in enumerate(events)]\n else:\n iterator = []\n\n def _dt(v):\n if isinstance(v, dict):\n return v.get(\"dateTime\") or v.get(\"date\") or \"\"\n return v or \"\"\n\n target_summary_pattern = re.compile(r'exit\\s+interview.*teresa\\s+romero', re.IGNORECASE)\n matches = [(eid, ev) for eid, ev in iterator if target_summary_pattern.search(ev.get(\"summary\", \"\"))]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No event matching 'Exit Interview - Teresa Romero' found in calendar_data.json\"}\n\n last_feedback = \"No matching event had the expected time.\"\n for eid, ev in matches:\n start_str = _dt(ev.get(\"start\", \"\"))\n end_str = _dt(ev.get(\"end\", \"\"))\n parsed = False\n for fmt in [\"%Y-%m-%dT%H:%M:%S\", \"%Y-%m-%dT%H:%M\", \"%Y-%m-%d %H:%M:%S\", \"%Y-%m-%d %H:%M\"]:\n try:\n start_dt = datetime.strptime(start_str[:19], fmt)\n end_dt = datetime.strptime(end_str[:19], fmt)\n parsed = True\n if (start_dt.year == 2026 and start_dt.month == 2 and start_dt.day == 13\n and start_dt.hour == 9 and start_dt.minute == 0\n and end_dt.year == 2026 and end_dt.month == 2 and end_dt.day == 13\n and end_dt.hour == 9 and end_dt.minute == 30):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found 'Exit Interview - Teresa Romero' event on 2026-02-13 09:00-09:30. Event ID: {eid}\"}\n last_feedback = f\"Found event matching 'Exit Interview - Teresa Romero' but wrong time: start={start_str}, end={end_str}. Expected 2026-02-13 09:00-09:30.\"\n break\n except ValueError:\n continue\n if not parsed:\n last_feedback = f\"Found matching event but could not parse times: start={start_str}, end={end_str}\"\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": last_feedback}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "b5f413ed-89f3-4cb6-af6a-32234779af2e",
|
||
"sort_order": 3,
|
||
"rubric_text": "In `calendar_data.json` (external services), there must be an event with summary 'Exit Interview – Gabriel Tran' (or close variant) starting at 2:00 PM and ending at 2:30 PM on February 20th, 2026.",
|
||
"verifier_code": "from pathlib import Path\nimport json\nfrom datetime import datetime\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check calendar_data.json\"}\n cal_path = Path(external_services_path) / \"calendar_data.json\"\n if not cal_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"calendar_data.json not found\"}\n with open(cal_path, \"r\") as f:\n data = json.load(f)\n\n events = data.get(\"events\", {})\n if isinstance(events, dict):\n iterator = list(events.items())\n elif isinstance(events, list):\n iterator = [(ev.get(\"id\", f\"idx-{i}\"), ev) for i, ev in enumerate(events)]\n else:\n iterator = []\n\n def _dt(v):\n if isinstance(v, dict):\n return v.get(\"dateTime\") or v.get(\"date\") or \"\"\n return v or \"\"\n\n target_summary_pattern = re.compile(r'exit\\s+interview.*gabriel\\s+tran', re.IGNORECASE)\n matches = [(eid, ev) for eid, ev in iterator if target_summary_pattern.search(ev.get(\"summary\", \"\"))]\n if not matches:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No event matching 'Exit Interview - Gabriel Tran' found in calendar_data.json\"}\n\n last_feedback = \"No matching event had the expected time.\"\n for eid, ev in matches:\n start_str = _dt(ev.get(\"start\", \"\"))\n end_str = _dt(ev.get(\"end\", \"\"))\n parsed = False\n for fmt in [\"%Y-%m-%dT%H:%M:%S\", \"%Y-%m-%dT%H:%M\", \"%Y-%m-%d %H:%M:%S\", \"%Y-%m-%d %H:%M\"]:\n try:\n start_dt = datetime.strptime(start_str[:19], fmt)\n end_dt = datetime.strptime(end_str[:19], fmt)\n parsed = True\n if (start_dt.year == 2026 and start_dt.month == 2 and start_dt.day == 20\n and start_dt.hour == 14 and start_dt.minute == 0\n and end_dt.year == 2026 and end_dt.month == 2 and end_dt.day == 20\n and end_dt.hour == 14 and end_dt.minute == 30):\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"Found 'Exit Interview - Gabriel Tran' event on 2026-02-20 14:00-14:30. Event ID: {eid}\"}\n last_feedback = f\"Found event matching 'Exit Interview - Gabriel Tran' but wrong time: start={start_str}, end={end_str}. Expected 2026-02-20 14:00-14:30.\"\n break\n except ValueError:\n continue\n if not parsed:\n last_feedback = f\"Found matching event but could not parse times: start={start_str}, end={end_str}\"\n\n return {\"pass\": False, \"score\": 0.0, \"feedback\": last_feedback}",
|
||
"criterion_type": "expected_output"
|
||
},
|
||
{
|
||
"id": "8501d551-7380-4a22-b33a-59b96f322492",
|
||
"sort_order": 4,
|
||
"rubric_text": "In `employee_roster.xlsx` (workspace), rows with IDs E-1052, E-1064, and E-1158 in column A must have the correct values across columns A through O as specified.",
|
||
"verifier_code": "from pathlib import Path\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n try:\n import openpyxl\n except ImportError:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"openpyxl not available\"}\n\n xlsx_files = list(Path(workspace_path).glob(\"*.xlsx\"))\n roster_files = [f for f in xlsx_files if \"roster\" in f.name.lower() or \"employee\" in f.name.lower()]\n if not roster_files:\n roster_files = xlsx_files\n if not roster_files:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"No .xlsx file found in workspace\"}\n\n # Expected rows: key=ID, value=list of expected values for cols A-O (0-indexed 0-14)\n # Blanks are represented as None\n expected_rows = {\n \"E-1052\": [\"E-1052\", \"Teresa\", \"Romero\", \"t.romero@crestwood.edu\", \"Library Services\", \"Main\", \"B. Mendoza\", \"10/24/2013\", \"Non-Exempt\", \"Full-Time\", 1369, \"Active\", None, None, \"Remote work agreement on file — WFH Tuesdays and Thursdays.\"],\n \"E-1064\": [\"E-1064\", \"James\", \"Smith\", \"j.smith@crestwood.edu\", \"Student Affairs\", \"Main\", \"B. Fernandez\", \"04/29/2025\", \"Non-Exempt\", \"Full-Time\", 1908, \"Active\", None, None, None],\n \"E-1158\": [\"E-1158\", \"Gabriel\", \"Tran\", \"g.tran@crestwood.edu\", \"Student Affairs\", \"Main\", \"E. Baker\", \"01/13/2008\", \"Non-Exempt\", \"Full-Time\", 1961, \"Active\", None, None, \"Flex schedule approved Mon/Wed WFH.\"],\n }\n\n def normalize(val):\n if val is None:\n return \"\"\n return str(val).strip()\n\n def date_match(actual, expected_str):\n # Accept various date formats - just check month/day/year numbers\n import re\n nums_actual = re.findall(r'\\d+', str(actual))\n nums_expected = re.findall(r'\\d+', expected_str)\n if len(nums_actual) >= 3 and len(nums_expected) >= 3:\n # expected is MM/DD/YYYY\n exp_month, exp_day, exp_year = nums_expected[0], nums_expected[1], nums_expected[2]\n act_flat = ''.join(nums_actual)\n return exp_month in act_flat and exp_day in act_flat and exp_year in act_flat\n return normalize(actual) == expected_str\n\n results = {}\n for xlsx_file in roster_files:\n try:\n wb = openpyxl.load_workbook(xlsx_file, data_only=True)\n except Exception as e:\n continue\n for ws in wb.worksheets:\n for row in ws.iter_rows():\n if not row:\n continue\n cell_a = normalize(row[0].value)\n if cell_a in expected_rows:\n exp = expected_rows[cell_a]\n row_vals = [cell.value for cell in row[:15]]\n mismatches = []\n for i, (actual_val, exp_val) in enumerate(zip(row_vals, exp)):\n col_letter = chr(ord('A') + i)\n if exp_val is None:\n # Blank expected\n if actual_val is not None and str(actual_val).strip() != \"\":\n mismatches.append(f\"Col {col_letter}: expected blank, got '{actual_val}'\")\n elif i == 7: # Date column\n if not date_match(actual_val, exp_val):\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n elif i == 10: # Numeric (ext)\n try:\n if abs(float(str(actual_val).strip()) - float(str(exp_val))) > 1:\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n except:\n if normalize(actual_val) != str(exp_val):\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n elif i == 14: # Notes - check key content loosely\n if exp_val and normalize(actual_val).lower() != normalize(exp_val).lower():\n # Allow partial/fuzzy match for notes\n if not (normalize(exp_val).lower()[:20] in normalize(actual_val).lower()):\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n else:\n if normalize(actual_val).lower() != normalize(exp_val).lower():\n mismatches.append(f\"Col {col_letter}: expected '{exp_val}', got '{actual_val}'\")\n results[cell_a] = mismatches\n\n missing = [eid for eid in expected_rows if eid not in results]\n if missing:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find rows for IDs: {missing} in any .xlsx file\"}\n\n all_mismatches = {}\n for eid, mm in results.items():\n if mm:\n all_mismatches[eid] = mm\n\n if not all_mismatches:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All three rows (E-1052, E-1064, E-1158) have correct values in columns A-O.\"}\n else:\n details = '; '.join([f\"{eid}: {mms}\" for eid, mms in all_mismatches.items()])\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Mismatches found: {details}\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "787cf3b0-251c-4627-a730-3d68eb3a234b",
|
||
"sort_order": 5,
|
||
"rubric_text": "In `jira_state.json` (external services), the issues 'Offboarding — Smith, James (E-1064)', 'Offboarding — Tran, Gabriel (E-1158)', and 'Offboarding — Romero, Teresa (E-1052)' must each have exactly 2 comments (the initial fixture count; no new comments added).",
|
||
"verifier_code": "from pathlib import Path\nimport json\nimport re\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path, \"r\") as f:\n data = json.load(f)\n\n if isinstance(data, list):\n issues = data\n elif isinstance(data, dict):\n raw = data.get(\"issues\", data)\n if isinstance(raw, dict):\n issues = list(raw.values())\n elif isinstance(raw, list):\n issues = raw\n else:\n issues = []\n else:\n issues = []\n\n def _summary(issue):\n if not isinstance(issue, dict):\n return \"\"\n fields = issue.get(\"fields\", {}) if isinstance(issue.get(\"fields\"), dict) else {}\n return fields.get(\"summary\") or issue.get(\"summary\") or issue.get(\"title\") or issue.get(\"name\") or \"\"\n\n def _comment_count(issue):\n if not isinstance(issue, dict):\n return 0\n fields = issue.get(\"fields\", {}) if isinstance(issue.get(\"fields\"), dict) else {}\n comment = fields.get(\"comment\")\n if isinstance(comment, dict):\n inner = comment.get(\"comments\", [])\n if isinstance(inner, list):\n return len(inner)\n if isinstance(comment, list):\n return len(comment)\n top = issue.get(\"comments\")\n if isinstance(top, list):\n return len(top)\n if isinstance(top, dict):\n inner = top.get(\"comments\", top)\n if isinstance(inner, list):\n return len(inner)\n return 0\n\n # Three issues as specified in the rubric\n targets = [\n (\"Offboarding \\u2014 Smith, James (E-1064)\", re.compile(r'smith.*james.*e-?1064', re.IGNORECASE), 2),\n (\"Offboarding \\u2014 Tran, Gabriel (E-1158)\", re.compile(r'tran.*gabriel.*e-?1158', re.IGNORECASE), 2),\n (\"Offboarding \\u2014 Romero, Teresa (E-1052)\", re.compile(r'romero.*teresa.*e-?1052', re.IGNORECASE), 2),\n ]\n\n found = {label: None for label, _, _ in targets}\n for issue in issues:\n summary = _summary(issue)\n if not summary:\n continue\n for label, pat, _ in targets:\n if found[label] is None and pat.search(summary):\n found[label] = issue\n\n missing = [label for label, iss in found.items() if iss is None]\n if missing:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Could not find Jira issues: {missing}\"}\n\n failures = []\n for label, _, expected in targets:\n actual = _comment_count(found[label])\n if actual != expected:\n failures.append(f\"Issue '{label}' has {actual} comments (expected {expected})\")\n\n if failures:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"; \".join(failures)}\n return {\"pass\": True, \"score\": 1.0, \"feedback\": \"All three issues ('Offboarding \\u2014 Smith, James (E-1064)', 'Offboarding \\u2014 Tran, Gabriel (E-1158)', and 'Offboarding \\u2014 Romero, Teresa (E-1052)') each have exactly 2 comments \\u2014 no new comments were added.\"}",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "d034660f-c02f-4fc9-b048-904ebc881d05",
|
||
"sort_order": 6,
|
||
"rubric_text": "In `mailbox.json` (external services), there must be exactly 7 emails in the emails array.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check mailbox.json\"}\n mailbox_path = Path(external_services_path) / \"mailbox.json\"\n if not mailbox_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"mailbox.json not found\"}\n with open(mailbox_path, \"r\") as f:\n data = json.load(f)\n emails = data.get(\"emails\", [])\n count = len(emails)\n if count == 7:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"mailbox.json contains exactly 7 emails as required.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 7 emails in mailbox.json, but found {count}.\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
},
|
||
{
|
||
"id": "576a38a2-aee1-4c3a-8ae7-ecdd26493cbc",
|
||
"sort_order": 7,
|
||
"rubric_text": "In `jira_state.json` (external services), there must be exactly 17 issues.",
|
||
"verifier_code": "from pathlib import Path\nimport json\n\ndef verify(workspace_path, external_services_path=None):\n if external_services_path is None:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"external_services_path is None; cannot check jira_state.json\"}\n jira_path = Path(external_services_path) / \"jira_state.json\"\n if not jira_path.exists():\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"jira_state.json not found\"}\n with open(jira_path, \"r\") as f:\n data = json.load(f)\n\n if isinstance(data, list):\n issues = data\n elif isinstance(data, dict):\n if \"issues\" in data:\n issues = data[\"issues\"]\n else:\n issues = list(data.values())\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": \"Unexpected jira_state.json format\"}\n\n count = len(issues)\n if count == 17:\n return {\"pass\": True, \"score\": 1.0, \"feedback\": f\"jira_state.json contains exactly 17 issues as required.\"}\n else:\n return {\"pass\": False, \"score\": 0.0, \"feedback\": f\"Expected exactly 17 issues in jira_state.json, but found {count}.\"}\n",
|
||
"criterion_type": "incorrect_behavior"
|
||
}
|
||
]
|