{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# TypeSafe Parallel Judgment Lab\n",
        "\n",
        "This notebook is the guided walkthrough and rerun companion for all **11 measured experiments**. It covers the recorded results, exact questions, inputs, formulas, retrieval rankings, agent trajectories, and runner commands.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "import json, os, subprocess, urllib.request\n",
        "\n",
        "def find_root():\n",
        "    here = Path.cwd()\n",
        "    for candidate in [here, *here.parents]:\n",
        "        if (candidate / \"public\" / \"experiments.json\").exists():\n",
        "            return candidate\n",
        "    raise FileNotFoundError(\"Open this notebook from the downloaded source bundle\")\n",
        "\n",
        "ROOT = find_root()\n",
        "DATA = json.loads((ROOT / \"public\" / \"experiments.json\").read_text())\n",
        "MANIFEST = json.loads((ROOT / \"experiments\" / \"RUN_MANIFEST.json\").read_text())\n",
        "print(\"Saved run:\", DATA[\"generated_at\"])\n",
        "print(\"Experiments:\", len(DATA[\"experiments\"]))\n",
        "print(\"Receipts:\", MANIFEST[\"totals\"])\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 1. Review the recorded run\n",
        "\n",
        "The table below is computed directly from the results file used by the website.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def receipt(e):\n",
        "    uses_nested = e[\"id\"] in {\"judge-grid\", \"agent-navigation\"}\n",
        "    m = e[\"typesafe\"][\"measured\"] if uses_nested else e[\"measured\"]\n",
        "    wall = e[\"typesafe\"][\"mean_wall_ms\"] if e[\"id\"] == \"judge-grid\" else m[\"wall_ms\"]\n",
        "    return {\"id\": e[\"id\"], \"calls\": m[\"calls\"], \"judgments\": m[\"judgments\"], \"wall_ms\": wall, \"cost\": m[\"usage\"][\"estimated_cost_usd\"]}\n",
        "\n",
        "rows = [receipt(e) for e in DATA[\"experiments\"]]\n",
        "for row in rows:\n",
        "    print(f'{row[\"id\"]:20} {row[\"calls\"]:4} calls  {row[\"judgments\"]:4} judgments  {row[\"wall_ms\"]:7.1f} ms  USD {row[\"cost\"]:.7f}')\n",
        "print(\"TOTAL\", sum(r[\"calls\"] for r in rows), \"calls /\", sum(r[\"judgments\"] for r in rows), \"judgments\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 2. Inspect any experiment\n",
        "\n",
        "This prints the exact question text and the first saved inputs/results. Retrieval experiments store their questions in the grid; every other experiment stores a question dictionary.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def inspect_experiment(experiment_id, sample_rows=3):\n",
        "    e = next(x for x in DATA[\"experiments\"] if x[\"id\"] == experiment_id)\n",
        "    questions = ({row[\"id\"]: row[\"label\"] for row in e[\"grid\"]} if e.get(\"kind\") == \"retrieval\" else e.get(\"questions\", {}))\n",
        "    print(\"TITLE:\", e[\"title\"])\n",
        "    print(\"DESCRIPTION:\", e[\"description\"])\n",
        "    print(\"\\nEXACT QUESTIONS:\")\n",
        "    for key, question in questions.items():\n",
        "        print(f\"- {key}: {question}\")\n",
        "    saved_rows = e.get(\"rows\") or e.get(\"documents\") or []\n",
        "    print(\"\\nSAVED INPUT/RESULT SAMPLES:\")\n",
        "    for row in saved_rows[:sample_rows]:\n",
        "        print(json.dumps(row, indent=2)[:2500], \"\\n\")\n",
        "    return e\n",
        "\n",
        "inspect_experiment(\"customer-voice\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 3. View all 11 setups\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "for e in DATA[\"experiments\"]:\n",
        "    q = e[\"grid\"] if e.get(\"kind\") == \"retrieval\" else e.get(\"questions\", {})\n",
        "    n_questions = len(q)\n",
        "    n_inputs = len(e.get(\"rows\") or e.get(\"documents\") or [])\n",
        "    print(f'{e[\"id\"]:20} {n_inputs:3} saved inputs/rows × {n_questions:2} questions')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 4. Make one live TypeSafe call\n",
        "\n",
        "Set `TYPESAFE_API_KEY` in your environment. This uses only Python's standard library. Question values can be shorthand Noul strings or complete Choice / Score specifications.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def call_typesafe(document, questions, model=\"speed_latest\"):\n",
        "    key = os.environ[\"TYPESAFE_API_KEY\"]\n",
        "    typed = {name: ({\"type\": \"noul\", \"instructions\": spec} if isinstance(spec, str) else spec) for name, spec in questions.items()}\n",
        "    body = json.dumps({\"document\": document, \"model\": model, \"questions\": typed}).encode()\n",
        "    request = urllib.request.Request(\n",
        "        \"https://api.typesafe.ai/v1/systemone\",\n",
        "        data=body,\n",
        "        headers={\"Authorization\": f\"Bearer {key}\", \"Content-Type\": \"application/json\"},\n",
        "        method=\"POST\",\n",
        "    )\n",
        "    with urllib.request.urlopen(request) as response:\n",
        "        return json.loads(response.read())\n",
        "\n",
        "# Uncomment to rerun a minimal Choice + Score example:\n",
        "# call_typesafe(\n",
        "#     \"SUBJECT: Contract expires at 3 PM\\nBODY: Please sign or tell us to walk away.\",\n",
        "#     {\n",
        "#       \"action\": {\"type\": \"choice\", \"instructions\": \"What should I do?\", \"criteria\": {\"reply_now\": \"Act immediately\", \"read_later\": \"No action today\"}},\n",
        "#       \"urgency\": {\"type\": \"score\", \"instructions\": \"How urgent is this?\", \"criteria\": [\"No action\", \"Can wait\", \"Today\", \"Immediate\"]},\n",
        "#     },\n",
        "# )\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 5. Rerun the complete lab\n",
        "\n",
        "These are the exact project runners. The first runner also uses `OPENAI_API_KEY` for the comparison judge. The agent runner needs Claude Headless. The QMD comparison needs QMD 2.8.3 and downloads its local embedding model on first use.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Run from the extracted source bundle after setting TYPESAFE_API_KEY and OPENAI_API_KEY.\n",
        "# subprocess.run([\"node\", \"scripts/run-experiments.mjs\"], cwd=ROOT, check=True)\n",
        "# subprocess.run([\"node\", \"scripts/run-agent-navigation.mjs\"], cwd=ROOT, check=True)\n",
        "# subprocess.run([\"node\", \"scripts/merge-agent-navigation.mjs\"], cwd=ROOT, check=True)\n",
        "# subprocess.run([\"node\", \"scripts/run-qmd-comparison.mjs\"], cwd=ROOT, check=True, env={**os.environ, \"QMD_BIN\": \"qmd\"})\n",
        "# subprocess.run([\"node\", \"scripts/run-ai-checker.mjs\", \"--reuse\"], cwd=ROOT, check=True)\n",
        "# subprocess.run([\"node\", \"scripts/run-more-experiments.mjs\"], cwd=ROOT, check=True)\n",
        "# subprocess.run([\"node\", \"scripts/build-downloads.mjs\"], cwd=ROOT, check=True)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 6. Check the data files\n",
        "\n",
        "The manifest hashes make changes to recorded experiment data visible.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import hashlib\n",
        "for relative, expected in MANIFEST[\"artifact_sha256\"].items():\n",
        "    actual = hashlib.sha256((ROOT / relative).read_bytes()).hexdigest()\n",
        "    print(\"PASS\" if actual == expected else \"FAIL\", relative, actual)\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}