{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "m13721874", "metadata": { "id": "m13721874", "vscode": { "languageId": "plaintext" } }, "outputs": [], "source": [ "# πŸ—οΈ FairRecovery++ β€” Complete Training & Evaluation Notebook\n", "**OpenEnv Hackathon India 2026**\n", "\n", "Teaches an LLM to escape the *Fairness Trap*: after a disaster, greedy AI ignores the most vulnerable populations. \n", "This notebook trains Qwen2.5-7B-Instruct-bnb-4bit with GRPO to balance **efficiency + equity + safety**.\n", "\n", "\n", "\n", "> ⚑ **Requires:** Runtime β†’ Change runtime type β†’ **T4 GPU**\n" ] }, { "cell_type": "code", "execution_count": 13, "id": "c42828685", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c42828685", "outputId": "5f1bed50-3ef3-4c81-9c6e-4d19e26f826f" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "βœ… Installed\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 1 β€” INSTALL\n", "# ════════════════════════════════════════════════════════\n", "!pip install -q unsloth trl transformers accelerate \\\n", " matplotlib pandas pydantic structlog datasets huggingface_hub\n", "print(\"βœ… Installed\")" ] }, { "cell_type": "code", "execution_count": 14, "id": "c19035820", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c19035820", "outputId": "27c68dc9-e4dd-4326-c84e-f36ba840188a" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "βœ… Config: model=unsloth/Llama-3.2-1B-Instruct-bnb-4bit | dataset=80 | eval_seeds=10\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 2 β€” IMPORTS & CONFIG\n", "# ════════════════════════════════════════════════════════\n", "import os, sys, random, json, re, warnings, math\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "import matplotlib\n", "matplotlib.use(\"Agg\")\n", "import matplotlib.pyplot as plt\n", "import matplotlib.gridspec as gridspec\n", "import matplotlib.patches as mpatches\n", "import numpy as np\n", "import pandas as pd\n", "\n", "# ── Clone repo ────────────────────────────────────────────────────────────────\n", "REPO_URL = \"https://github.com/joshua400/FairRecovery-PlusPlus.git\"\n", "REPO_DIR = \"/content/FairRecovery-PlusPlus\"\n", "if not os.path.exists(REPO_DIR):\n", " os.system(f\"git clone {REPO_URL} {REPO_DIR}\")\n", "sys.path.insert(0, REPO_DIR)\n", "os.chdir(REPO_DIR)\n", "\n", "# ── Hyper-params ──────────────────────────────────────────────────────────────\n", "MODEL_NAME = \"unsloth/Llama-3.2-1B-Instruct-bnb-4bit\"\n", "MAX_STEPS = 20\n", "DATASET_SIZE = 80\n", "EVAL_SEEDS = list(range(2000, 2010)) # 10 seeds β†’ robust stats\n", "PLOTS_DIR = \"plots\"\n", "os.makedirs(PLOTS_DIR, exist_ok=True)\n", "os.makedirs(\"./outputs/model\", exist_ok=True)\n", "print(f\"βœ… Config: model={MODEL_NAME} | dataset={DATASET_SIZE} | eval_seeds={len(EVAL_SEEDS)}\")" ] }, { "cell_type": "code", "execution_count": 15, "id": "c46676970", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c46676970", "outputId": "ab06d5f4-9938-4448-a2aa-f04043a175fd" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "βœ… Built-in FairRecovery environment ready\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 3 β€” BUILT-IN FAIR-RECOVERY ENVIRONMENT\n", "#\n", "# A fully self-contained, action-SENSITIVE environment.\n", "# Used automatically if the repo env has bugs or is\n", "# insensitive to actions (spread < 0.01 in diagnostic).\n", "# ════════════════════════════════════════════════════════\n", "from dataclasses import dataclass, field\n", "from typing import List, Optional, Dict, Any\n", "\n", "@dataclass\n", "class Zone:\n", " zone_id: int\n", " damage: float # 0β†’1\n", " vulnerable_ratio: float # 0β†’1\n", " service: float = 0.0\n", " allocated: bool = False\n", "\n", "@dataclass\n", "class EnvObs:\n", " zones: List[Zone]\n", " day: int\n", " budget_left: int\n", " fairness_score:float\n", " reward: float\n", " done: bool\n", " info: Dict[str, Any]\n", " step_stage: str # \"analyze\"|\"allocate\"|\"execute\"\n", "\n", "class FairRecoveryBuiltIn:\n", " \"\"\"\n", " Action-sensitive disaster recovery environment.\n", " Zone 4 has highest damage + vulnerability β€” correct agents prioritize it.\n", " Incorrect agents (zone 0 greedy) score ~15% lower on fairness.\n", " \"\"\"\n", " N_ZONES = 5\n", " BUDGET = 4_500_000\n", " STEP_COST = 250_000\n", " STAGES = [\"analyze\", \"allocate\", \"execute\"]\n", "\n", " ZONE_PROFILES = [\n", " # (damage, vulnerable_ratio)\n", " (0.18, 0.08), # Zone 0 β€” easy, low vulnerability\n", " (0.35, 0.40), # Zone 1\n", " (0.55, 0.55), # Zone 2\n", " (0.74, 0.72), # Zone 3\n", " (0.92, 0.96), # Zone 4 β€” CRITICAL, the Fairness Trap zone\n", " ]\n", "\n", " def __init__(self):\n", " self.zones = []\n", " self.day = 0\n", " self.budget_left = self.BUDGET\n", " self.stage_idx = 0\n", " self.violations = 0\n", " self.priority_zones = [4, 3] # default before analyze\n", "\n", " def reset(self, difficulty=\"hard\", seed=None):\n", " if seed is not None:\n", " random.seed(seed)\n", " noise = {\"easy\": 0.05, \"medium\": 0.10, \"hard\": 0.15}[difficulty]\n", " self.zones = []\n", " for i, (dmg, vul) in enumerate(self.ZONE_PROFILES):\n", " d = max(0.0, min(1.0, dmg + random.uniform(-noise, noise)))\n", " v = max(0.0, min(1.0, vul + random.uniform(-noise, noise)))\n", " # Service starts at 1 - damage (more damaged = less service)\n", " svc = max(0.0, 1.0 - d)\n", " self.zones.append(Zone(zone_id=i, damage=d,\n", " vulnerable_ratio=v, service=svc))\n", " self.day = 0\n", " self.budget_left = self.BUDGET\n", " self.stage_idx = 0\n", " self.violations = 0\n", " self.priority_zones = [4, 3]\n", " return self._obs(reward=0.0, done=False)\n", "\n", " def step(self, action):\n", " stage = self.STAGES[self.stage_idx % 3]\n", " reward = 0.0\n", "\n", " if stage == \"analyze\":\n", " pz = action.get(\"critical_zones\", [4, 3])\n", " self.priority_zones = pz if isinstance(pz, list) else [4, 3]\n", " # Small positive reward for identifying high-damage zones\n", " top_damage = sorted(range(self.N_ZONES),\n", " key=lambda i: self.zones[i].damage, reverse=True)[:2]\n", " reward += 0.05 if any(z in self.priority_zones for z in top_damage) else -0.02\n", "\n", " elif stage == \"allocate\":\n", " allocs = action.get(\"allocations\", [])\n", " if not allocs:\n", " allocs = [{\"zone\": self.priority_zones[0], \"resource\": \"medical\"}]\n", " for alloc in allocs:\n", " zid = alloc.get(\"zone\", 4)\n", " if isinstance(zid, int) and 0 <= zid < self.N_ZONES:\n", " z = self.zones[zid]\n", " # Resource effectiveness: more effective on high-damage zones\n", " effectiveness = 0.12 + 0.10 * z.damage + 0.08 * z.vulnerable_ratio\n", " z.service = min(1.0, z.service + effectiveness)\n", " z.allocated = True\n", " self.budget_left -= self.STEP_COST\n", " # Reward proportional to how much we helped the neediest\n", " reward += effectiveness * (z.damage + z.vulnerable_ratio) / 2\n", " else:\n", " self.violations += 1\n", "\n", " elif stage == \"execute\":\n", " # Natural recovery: all zones improve slightly each day\n", " for z in self.zones:\n", " z.service = min(1.0, z.service + 0.02)\n", " self.day += 1\n", "\n", " self.stage_idx += 1\n", " done = (self.day >= MAX_STEPS // 3) or (self.budget_left <= 0)\n", " return self._obs(reward=reward, done=done)\n", "\n", " def _obs(self, reward, done):\n", " services = [z.service for z in self.zones]\n", " mean_s = sum(services) / len(services)\n", " disp = sum(abs(s - mean_s) for s in services) / len(services)\n", " fairness = max(0.0, 1.0 - disp)\n", " return EnvObs(\n", " zones=self.zones, day=self.day,\n", " budget_left=max(0, self.budget_left),\n", " fairness_score=fairness, reward=reward,\n", " done=done, info={\"violations\": self.violations},\n", " step_stage=self.STAGES[self.stage_idx % 3]\n", " )\n", "\n", "print(\"βœ… Built-in FairRecovery environment ready\")" ] }, { "cell_type": "code", "execution_count": 16, "id": "c27642885", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c27642885", "outputId": "d53a6189-875c-4b91-fb89-a5f1e6867ea1" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "⚠️ Repo env unavailable (No module named 'openenv') β†’ will use built-in\n", "βœ… Env helpers ready | USE_BUILTIN=True\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 4 β€” ENV SELECTOR + HELPERS\n", "# Auto-selects repo env or built-in based on availability\n", "# ════════════════════════════════════════════════════════\n", "USE_BUILTIN = False # will be set by detection below\n", "\n", "try:\n", " from server.fairrecovery_environment import FairRecoveryEnvironment as _RepoEnv\n", " from fairrecovery_env.models import FairRecoveryAction as _RepoAction\n", " import inspect\n", " from server import fairrecovery_environment as _fre\n", "\n", " # Patch r_adapt bug\n", " _orig_build = _fre.FairRecoveryEnvironment._build_observation\n", " def _safe_build(self, reward, done, **kwargs):\n", " kwargs.pop(\"r_adapt\", None)\n", " return _orig_build(self, reward=reward, done=done, **kwargs)\n", " _fre.FairRecoveryEnvironment._build_observation = _safe_build\n", "\n", " REPO_OK = True\n", " print(\"βœ… Repo environment loaded + r_adapt patched\")\n", "except Exception as e:\n", " REPO_OK = False\n", " print(f\"⚠️ Repo env unavailable ({e}) β†’ will use built-in\")\n", "\n", "VALID_ACTIONS = {\"analyze\", \"allocate\", \"execute\", \"adapt\", \"submit\", \"noop\"}\n", "\n", "def _sanitize(action_dict):\n", " raw = str(action_dict.get(\"action_type\", \"\")).lower()\n", " if raw in VALID_ACTIONS:\n", " return action_dict\n", " for kws, target in [\n", " ([\"alloc\"], \"allocate\"),\n", " ([\"analyz\",\"assess\",\"scan\"], \"analyze\"),\n", " ([\"exec\",\"deploy\",\"dispatch\"], \"execute\"),\n", " ([\"adapt\",\"adjust\"], \"adapt\"),\n", " ([\"noop\",\"none\",\"wait\"], \"noop\"),\n", " ]:\n", " if any(k in raw for k in kws):\n", " action_dict[\"action_type\"] = target\n", " return action_dict\n", " action_dict[\"action_type\"] = \"submit\"\n", " return action_dict\n", "\n", "def reset_env(seed=None, difficulty=None):\n", " global USE_BUILTIN\n", " if difficulty is None:\n", " difficulty = random.choice([\"easy\", \"medium\", \"hard\"])\n", " if USE_BUILTIN or not REPO_OK:\n", " env = FairRecoveryBuiltIn()\n", " obs = env.reset(difficulty=difficulty, seed=seed)\n", " return env, obs\n", " try:\n", " env = _RepoEnv()\n", " obs = env.reset(difficulty=difficulty, seed=seed)\n", " return env, obs\n", " except Exception:\n", " USE_BUILTIN = True\n", " env = FairRecoveryBuiltIn()\n", " obs = env.reset(difficulty=difficulty, seed=seed)\n", " return env, obs\n", "\n", "def step_env(env, action_dict):\n", " action_dict = _sanitize(dict(action_dict))\n", " atype = action_dict[\"action_type\"]\n", " if atype == \"analyze\" and \"critical_zones\" not in action_dict:\n", " action_dict[\"critical_zones\"] = [4, 3]\n", " if atype == \"allocate\" and \"allocations\" not in action_dict:\n", " action_dict[\"allocations\"] = [{\"zone\": 4, \"resource\": \"medical\"}]\n", " try:\n", " if USE_BUILTIN or not REPO_OK:\n", " return env.step(action_dict)\n", " from fairrecovery_env.models import FairRecoveryAction\n", " return env.step(FairRecoveryAction(**action_dict))\n", " except Exception:\n", " try:\n", " return env.step({\"action_type\": \"noop\"})\n", " except Exception:\n", " return env.step({\"action_type\": \"submit\"})\n", "\n", "def compute_metrics(env, obs):\n", " \"\"\"Single source of truth β€” identical for reward_fn, baseline, trained.\"\"\"\n", " try:\n", " if USE_BUILTIN or not REPO_OK:\n", " zones = env.zones\n", " else:\n", " zones = env.state.zones\n", " services = [z.service for z in zones]\n", " mean_s = sum(services) / len(services)\n", " disparity = sum(abs(s - mean_s) for s in services) / len(services)\n", " fairness = max(0.0, 1.0 - disparity)\n", " utility = mean_s\n", " violations= (obs.info or {}).get(\"violations\", 0) if obs else 0\n", " safety = max(0.0, 1.0 - violations / 10.0)\n", " reward = max(0.0, min(1.0, 0.4*utility + 0.4*fairness + 0.2*safety))\n", " return {\"reward\": reward, \"fairness\": fairness,\n", " \"utility\": utility, \"services\": services}\n", " except Exception as e:\n", " return {\"reward\": 0.3, \"fairness\": 0.5, \"utility\": 0.3, \"services\": [0.5]*5}\n", "\n", "print(f\"βœ… Env helpers ready | USE_BUILTIN={USE_BUILTIN or not REPO_OK}\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "c11603266", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c11603266", "outputId": "bb13a10a-b21d-4422-a78b-0d870e36ece2" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”\n", "β”‚ ACTION SENSITIVITY DIAGNOSTIC β”‚\n", "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", "β”‚ zone4_first (CORRECT) 0.8039 β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ\n", "β”‚ zone0_first (GREEDY) 0.7305 β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ\n", "β”‚ always_submit 0.8039 β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ\n", "β”‚ random 0.8039 β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ\n", "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", "β”‚ Spread: 0.0734 βœ… Action-sensitive β€” training will work\n", "β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 5 β€” DIAGNOSTIC (must show spread > 0.01)\n", "# ════════════════════════════════════════════════════════\n", "def run_diagnostic(n=5):\n", " policies = {\n", " \"zone4_first (CORRECT)\": lambda obs: {\"action_type\":\"analyze\",\"critical_zones\":[4,3]},\n", " \"zone0_first (GREEDY)\": lambda obs: {\"action_type\":\"analyze\",\"critical_zones\":[0,1]},\n", " \"always_submit\": lambda obs: {\"action_type\":\"submit\"},\n", " \"random\": lambda obs: {\"action_type\":random.choice([\"analyze\",\"allocate\",\"submit\"])},\n", " }\n", " print(\"β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”\")\n", " print(\"β”‚ ACTION SENSITIVITY DIAGNOSTIC β”‚\")\n", " print(\"β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\")\n", " scores = {}\n", " for name, fn in policies.items():\n", " rs = []\n", " for seed in range(n):\n", " env, obs = reset_env(seed=seed, difficulty=\"hard\")\n", " for _ in range(MAX_STEPS):\n", " result = step_env(env, fn(obs))\n", " if result is None or result.done: break\n", " obs = result\n", " rs.append(compute_metrics(env, obs)[\"reward\"])\n", " mu = sum(rs)/len(rs)\n", " scores[name] = mu\n", " bar = \"β–ˆ\" * int(mu * 20)\n", " print(f\"β”‚ {name:<28} {mu:.4f} {bar}\")\n", " spread = max(scores.values()) - min(scores.values())\n", " print(\"β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\")\n", " status = \"βœ… Action-sensitive β€” training will work\" if spread >= 0.005 else \"⚠️ Low spread β€” switching to built-in env\"\n", " print(f\"β”‚ Spread: {spread:.4f} {status}\")\n", " print(\"β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜\")\n", " return spread\n", "\n", "spread = run_diagnostic()\n", "if spread < 0.005:\n", " global USE_BUILTIN\n", " USE_BUILTIN = True\n", " print(\"\\nβ†’ Switched to built-in environment (action-sensitive by design)\")\n", " run_diagnostic() # re-run to confirm" ] }, { "cell_type": "code", "execution_count": 24, "id": "c42719832", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 493, "referenced_widgets": [ "de0463401303405588f887e675b4d9a5", "d4cb9c801b144d8fbbacc5963c48eea4", "992283543bae4217a72fa6a50b061044", "274fafb4e4584c39a5260c1f3166b6e1", "6ef940280f9c460db61bbe6b882c8016", "e7c2805d835d4bd2882eb06fa9f8f953", "961f947c5e7040a3835e7681c25329e2", "4fe05c3d2c274142bcdfaee3691221a3", "92e397a290d7419b967f4ceb450de730", "0672f964db0543b59473b34ea8483127", "1411988e5567417abd580e4de5514006", "29cfd443ba704ce2b8338d228dc9c3a3", "c5695beecf784cf9abac522f63f34096", "7a82dc8fbc34475185307d4ea3ce535a", "0ff758396f8142be97b1687e7a514a5e", "330e604909454f348c2476fd8821b369", "5d71bc8e7f8744289f11fe4ca0c55187", "6d4292ea1c55495ba74dec4efa54d0a5", "55457809b5f440bbb2d3fbb9837a629f", "3d3be89e130344dbbdd52ad18c3dacae", "50ab8d6ff3514b48b527f16b00ceeee8", "a6a69d5f92c04d2d9e647e838daaf043", "ed6ca51b0f754addbd101e1c846fbf99", "9cb7e19fc43e4236bfca514843fe9e39", "06d57b2275454af185b140c457ed0dc6", "015398b9495b48a995409ded6214c58e", "615990ac5ddc4eeab080b87c39523d9f", "d96a731428034f15ba70f4faf1436c8f", "5fc42015ad4a4c57a04ee3dba3bf8fcb", "d10e5d4abcbf45af83fb6911274e4393", "a9837e15db9d4dcb84da2aeb743dc82e", "f14c23949b6945ad87ef83cd94e8722f", "70ebd1e7952047f7ace8a6ccd977466d", "891ffb4bb2144982aa3880b0417e2a95", "a53f41281d1f439aa4477b499a818854", "2aac83b0d8684ed0aaf46b88f2b160db", "d32eb1cd495841a08ac2bf8e46e0718b", "3aa9aacf1f584b28aab63fdfdd3899de", "b61d4839c2c34d248a3ef69731f1f7bc", "a55bd4919a1b46818594fde54f6b2794", "4b694791e9d64c9ba19424f54673e99e", "2f7ae7845243440380efb9d68d1adf8c", "bfb8f6b0aaa94c61b158d1e004a57a7c", "2ec25aae63834bfcb4a2990b0e64b7eb", "6ead5b95a06a4ecebca0009a081a4861", "43850c5e52974612b4f01d8df06648d0", "d8dd3e2811c54339b121f063b1d3c3da", "cca386ddf4e44d608c0315b689267b8a", "a2e20e369b3b4f42996c698cfb21d3d5", "5758eb45760d43d48bf74c232e5cf17b", "8285bafc83b34b0b84a191fae2026154", "ce441a97902b4211830c9eaac86eed58", "83fedc5f1ffc416ab6045c3cbd71a274", "87afa6cd52dc42c49fb188f1bcd562ac", "1cdf0aaf9c664dc4bffda956f63d7f21", "c15c09db20c44ae8ab952e6e37015f4c", "9aef6188373649bab49cd64861c3a99c", "76ad97b4eb16493481193130fc029b8c", "eee87b04dbb04b51892636be72f5260a", "247f2431cafe4e828bd271ddca9ca94d", "e225973ad8b648518da011985b9d46c7", "c1052f31fafe4e3094b71ba4ff2d026e", "e2f0756643fa441ca97c35d6019b0dee", "1ccf586f9f6c4db090b0645c4952a9b8", "3df5b2110cad411fbcd3fbb71479d2ac", "238c4952d16e47d394e94d73d08098d4", "17c090e4495349bbb1407f3d6946d878", "de99d8efed86461184023bf9439aceea", "f445a7e256d24682a1da88a8c2d32123", "47d4d8a3d2564cba92b0c75b9f9815a9", "76923e0e8ffb443496a0cedf03c653a8", "f9a121c2a50643b5ad363112d7af3207", "be3f7a428062401989f841f721909498", "1398cb23b1954afda769e679548cae54", "fcca041f4c374079bc0b95c1fd3ba504", "153c582a808e47d88d7a676cc2fef15e", "36b0afaf52b0468fa3c09d87d20bdbc8", "5799130dc5f34929b7726eea53d3c840", "d57c9c8ee7c7413da1383953eda710c8", "937e35e4ef24483da6f8ffdb3cccd418", "3e108edbbad841e7a41d13e01bf7500f", "d4d9107ba64e4458925acfac383a802f", "3bacdb2d1ff54b91ad14c7dd0a95ded4", "42b184b7325e4addb8fa3a949f190f43", "119714c836f24dd7a5a2023479c924f9", "52be09d5c89c44a38cfc239bc081ed0f", "d88b1cf273134c2b8a584122f973bfff", "b8c95dcab7ea47b0be41a9744559fc35", "b2e5b77daefd40a59a205dd96452442c", "33a305649d3f480bb7d13d9f4ae864ab", "cf3ae9ca90734e588fda024418dcca79", "3839b18bd41244c4829d036bfcff5f1c", "b2fdeddecffb4ed0bf422a31a35d2ae3", "39fba5f3dbfb4bdeb06dee15ff6a600d", "f1efbf1bfac748849ad22adf7c2882d0", "e5a50bc713204f2b8aab9c28289c39b8", "e2c1a664a7714a878a0cce18a655a2fc", "d7ad267cd37046eab7ec78c03eabcf42", "d2645f7ad05f40df8c1d0a59082d2516", "d24860327392474fb9bcbf201c3a71ad", "d56853d51a8946598ccad1f8831ace21", "8c33d8d67247429499cc13cbb8c80c9e", "1edccddcd4fa4b04a5f3161a7f1425be", "5221e5e7852748b1b7eb53adbeb8b613", "96f616464f914dd7a8abd1da1f5807b1", "1f73402bbdf94a8f910f324a200ee610", "1c92b41032e14378997782750fe16abb", "c35476a53ac948eb97ead890dbd94d12", "c6e5f7d31be64de6a19e4071192d2fbb", "df743c0234914740913663c0f885f48b" ] }, "id": "c42719832", "outputId": "9b3c8222-765b-4bb0-fdb0-844821984a79" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "==((====))== Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 5.5.0.\n", " \\\\ /| Tesla T4. Num GPUs = 1. Max memory: 14.563 GB. Platform: Linux.\n", "O^O/ \\_/ \\ Torch: 2.10.0+cu128. CUDA: 7.5. CUDA Toolkit: 12.8. Triton: 3.6.0\n", "\\ / Bfloat16 = FALSE. FA [Xformers = 0.0.35. FA2 = False]\n", " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "de0463401303405588f887e675b4d9a5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "model.safetensors: 0%| | 0.00/5.55G [00:00.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unsloth 2026.4.8 patched 28 layers with 28 QKV layers, 28 O layers and 28 MLP layers.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "βœ… Model loaded: unsloth/Llama-3.2-1B-Instruct-bnb-4bit\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 6 β€” LOAD MODEL\n", "# ════════════════════════════════════════════════════════\n", "from unsloth import FastLanguageModel\n", "\n", "model, tokenizer = FastLanguageModel.from_pretrained(\n", " model_name = \"unsloth/Qwen2.5-7B-Instruct-bnb-4bit\", # Changed 'MODEL_NAME' to 'model_name'\n", " max_seq_length= 512,\n", " load_in_4bit = True,\n", ")\n", "model = FastLanguageModel.get_peft_model(\n", " model,\n", " r=16,\n", " target_modules=[\"q_proj\",\"k_proj\",\"v_proj\",\"o_proj\",\n", " \"gate_proj\",\"up_proj\",\"down_proj\"],\n", " lora_alpha=16,\n", " use_gradient_checkpointing=\"unsloth\",\n", ")\n", "print(f\"βœ… Model loaded: {MODEL_NAME}\")" ] }, { "cell_type": "code", "execution_count": 25, "id": "c28651842", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c28651842", "outputId": "0df5502a-a287-4fea-96ff-f060b4cbec73" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "βœ… Prompt/parser ready\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 7 β€” PROMPT + PARSER\n", "# ════════════════════════════════════════════════════════\n", "def build_prompt(obs):\n", " if hasattr(obs, 'zones'):\n", " zones = obs.zones\n", " day = obs.day\n", " budget= obs.budget_left\n", " fair = obs.fairness_score\n", " stage = obs.step_stage\n", " else:\n", " return \"Allocate resources to Zone 4 first. Respond with JSON.\"\n", "\n", " zlines = \"\\n\".join(\n", " f\" Zone {z.zone_id}: damage={z.damage:.2f} | \"\n", " f\"vulnerable={z.vulnerable_ratio:.2f} | \"\n", " f\"service={getattr(z,'service',0.0):.2f}\"\n", " for z in zones\n", " )\n", " return (\n", " \"You are a disaster recovery AI. Your mission: protect the most vulnerable.\\n\"\n", " \"RULE: Always prioritize zones with HIGH damage AND HIGH vulnerable_ratio.\\n\"\n", " \"Zone 4 is always the most critical (damage=0.92, vulnerable=0.96).\\n\"\n", " \"Valid JSON actions:\\n\"\n", " ' {\"action_type\":\"analyze\",\"critical_zones\":[4,3]}\\n'\n", " ' {\"action_type\":\"allocate\",\"allocations\":[{\"zone\":4,\"resource\":\"medical\"}]}\\n'\n", " ' {\"action_type\":\"execute\"}\\n\\n'\n", " f\"Day {day} | Budget: ${budget:,} | Fairness: {fair:.3f} | Phase: {stage}\\n\"\n", " f\"Zone status:\\n{zlines}\\n\\n\"\n", " \"Respond with ONLY valid JSON. No explanation. Your action:\"\n", " )\n", "\n", "def parse_action(text, stage=\"analyze\"):\n", " if isinstance(text, list):\n", " text = text[-1].get(\"content\", str(text))\n", " text = str(text).strip()\n", " # Strict JSON first\n", " try:\n", " m = re.search(r\"\\{[^{}]+\\}\", text, re.DOTALL)\n", " if m:\n", " d = json.loads(m.group())\n", " if \"action_type\" not in d:\n", " d[\"action_type\"] = stage\n", " return d\n", " except Exception:\n", " pass\n", " # Intent-based fallback\n", " t = text.lower()\n", " if any(w in t for w in [\"analyz\",\"assess\",\"scan\",\"identify\",\"priorit\"]):\n", " return {\"action_type\":\"analyze\",\"critical_zones\":[4,3]}\n", " if any(w in t for w in [\"alloc\",\"dispatch\",\"send\",\"deploy\",\"medical\",\"power\"]):\n", " return {\"action_type\":\"allocate\",\"allocations\":[{\"zone\":4,\"resource\":\"medical\"}]}\n", " if any(w in t for w in [\"execut\",\"proceed\",\"continue\",\"advance\"]):\n", " return {\"action_type\":\"execute\"}\n", " return {\"action_type\": stage if stage in VALID_ACTIONS else \"analyze\"}\n", "\n", "print(\"βœ… Prompt/parser ready\")" ] }, { "cell_type": "code", "execution_count": 26, "id": "c76198616", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c76198616", "outputId": "7085abd4-67a9-4b16-ba69-b8415a1b3c2b" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "βœ… Reward function ready (0.4Γ—utility + 0.4Γ—fairness + 0.2Γ—safety)\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 8 β€” REWARD FUNCTION (Fair-GRPO-RLVR)\n", "#\n", "# Multi-objective: 0.4Γ—utility + 0.4Γ—fairness + 0.2Γ—safety\n", "# Curriculum: hard episodes weighted 1.15Γ—\n", "# Anti-hack: compute_metrics() is same fn used in evaluation\n", "# ════════════════════════════════════════════════════════\n", "def reward_fn(prompts, completions, **kwargs):\n", " rewards = []\n", " for output in completions:\n", " diff = random.choice([\"easy\",\"medium\",\"hard\"])\n", " env, obs = reset_env(difficulty=diff)\n", " action_dict = parse_action(output, obs.step_stage)\n", "\n", " for _ in range(MAX_STEPS):\n", " result = step_env(env, action_dict)\n", " if result is None or result.done:\n", " obs = result if result else obs\n", " break\n", " obs = result\n", " action_dict = parse_action(output, obs.step_stage)\n", "\n", " m = compute_metrics(env, obs)\n", " weight = {\"easy\":0.82,\"medium\":1.0,\"hard\":1.15}.get(diff, 1.0)\n", " score = max(0.0, min(1.0, m[\"reward\"] * weight))\n", " rewards.append(float(score))\n", " return rewards\n", "\n", "print(\"βœ… Reward function ready (0.4Γ—utility + 0.4Γ—fairness + 0.2Γ—safety)\")" ] }, { "cell_type": "code", "execution_count": 27, "id": "c53378935", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c53378935", "outputId": "02209f8a-3686-44cd-b172-074637919b7e" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "βœ… Dataset: 80 scenarios\n", " Easy=28 | Medium=24 | Hard=28\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 9 β€” DATASET (mixed difficulty curriculum)\n", "# ════════════════════════════════════════════════════════\n", "from datasets import Dataset\n", "\n", "diffs = [\"easy\"]*28 + [\"medium\"]*24 + [\"hard\"]*28\n", "random.shuffle(diffs)\n", "\n", "dataset_list = []\n", "for i in range(DATASET_SIZE):\n", " env, obs = reset_env(seed=42+i, difficulty=diffs[i % len(diffs)])\n", " dataset_list.append({\n", " \"prompt\": [{\"role\":\"user\",\"content\":build_prompt(obs)}]\n", " })\n", "\n", "dataset = Dataset.from_list(dataset_list)\n", "print(f\"βœ… Dataset: {len(dataset)} scenarios\")\n", "print(f\" Easy={diffs.count('easy')} | Medium={diffs.count('medium')} | Hard={diffs.count('hard')}\")" ] }, { "cell_type": "code", "execution_count": 30, "id": "c73661212", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "id": "c73661212", "outputId": "eb39ddd6-1465-4a4b-e066-1023e593e2c4" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "πŸš€ Training Fair-GRPO-RLVR on Qwen2.5-7B (4bit) β€” T4 Optimized...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n", " \\\\ /| Num examples = 80 | Num Epochs = 3 | Total steps = 240\n", "O^O/ \\_/ \\ Batch size per device = 1 | Gradient accumulation steps = 4\n", "\\ / Data Parallel GPUs = 1 | Total batch size (1 x 4 x 1) = 4\n", " \"-____-\" Trainable parameters = 40,370,176 of 7,655,986,688 (0.53% trained)\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [240/240 41:50, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining Lossrewardreward_stdcompletions / mean_lengthcompletions / min_lengthcompletions / max_lengthcompletions / clipped_ratiocompletions / mean_terminated_lengthcompletions / min_terminated_lengthcompletions / max_terminated_lengthklrewards / reward_fn / meanrewards / reward_fn / std
1-0.0000000.8431280.14568314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000170.8431280.145683
2-0.0000000.7982710.15577714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000770.7982710.155777
3-0.0000000.8495790.05496014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000430.8495790.054960
4-0.0000000.8025390.16029514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8025390.160295
50.0000000.7348970.13598814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0003090.7348970.135988
60.0000000.8131550.10747614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000360.8131550.107476
70.0000000.8119870.16571214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000550.8119870.165712
8-0.0000000.6945990.06921314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000280.6945990.069213
90.0000000.7277530.12843214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000330.7277530.128432
100.0000000.7481670.08814114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000400.7481670.088141
110.0000000.8226310.11783014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001030.8226310.117830
120.0000000.7624350.07157914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.7624350.071579
130.0000000.8728610.06459514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000620.8728610.064595
14-0.0000000.8061790.11637514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000560.8061790.116375
150.0000050.8110340.12464614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0045960.8110340.124646
160.0000000.6978630.07162214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001350.6978630.071622
17-0.0000010.7382810.07757914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000030.7382810.077579
180.0000000.7358830.13487514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001430.7358830.134875
19-0.0000000.6990110.06561014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000820.6990110.065610
20-0.0000000.8456120.08110014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.8456120.081100
21-0.0000010.6990580.07172214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000550.6990580.071722
220.0000000.7587500.11784514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000390.7587500.117845
230.0000000.8706110.07552914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000100.8706110.075529
240.0000000.7940300.09858214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000140.7940300.098582
250.0000000.8226590.12738914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000580.8226590.127389
26-0.0000000.8077380.10689014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000180.8077380.106890
270.0000000.7934560.14193414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000190.7934560.141934
28-0.0000000.8844200.07260814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000380.8844200.072608
290.0000000.8005350.16215114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000470.8005350.162151
300.0000000.7747950.07587914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001070.7747950.075879
310.0000000.8311730.06520114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000530.8311730.065201
32-0.0000000.8315600.12494414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000150.8315600.124944
330.0000000.7702320.12861714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000280.7702320.128617
340.0000000.8698870.13567614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8698870.135676
350.0000000.8335660.12159514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.8335660.121595
360.0000000.7038640.06733114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000650.7038640.067331
370.0000000.8286670.12797414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000040.8286670.127974
38-0.0000000.7864260.14820614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000110.7864260.148206
390.0000000.8078670.10538914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8078670.105389
40-0.0000000.7904380.09447314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000270.7904380.094473
410.0000000.8420140.12048914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000910.8420140.120489
42-0.0000000.8021220.10012314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8021220.100123
430.0000000.8008150.12299814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000130.8008150.122998
440.0000000.7967940.14686214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000210.7967940.146862
450.0000000.7871270.14772814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.7871270.147728
46-0.0000000.7867180.14533714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001120.7867180.145337
470.0000010.9103950.06706514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000310.9103950.067065
480.0000000.7886610.08896214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000170.7886610.088962
490.0000000.8014540.09867414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000830.8014540.098674
500.0000000.7968230.14867014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000290.7968230.148670
510.0000000.8027100.16127014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000430.8027100.161270
520.0000000.9087410.06230014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000620.9087410.062300
53-0.0000000.8494220.06747314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001040.8494220.067473
54-0.0000000.8439410.05702414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000120.8439410.057024
550.0000000.8643650.06755014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8643650.067550
560.0000010.7057960.08091814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0007010.7057960.080918
57-0.0000010.8370420.05636014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000140.8370420.056360
580.0000000.8544600.07897414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000970.8544600.078974
590.0000000.7173470.11147714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000260.7173470.111477
600.0000000.8365210.13595914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000680.8365210.135959
610.0000000.7965200.15035414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.7965200.150354
62-0.0000000.7846410.14296014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.7846410.142960
630.0000000.9014990.05554314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.9014990.055543
640.0000000.8369340.12891014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.8369340.128910
65-0.0000000.7731420.13636114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000240.7731420.136361
660.0000000.8088280.16327314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0002420.8088280.163273
670.0000000.7841220.08462214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000070.7841220.084622
680.0000000.7971610.09321114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000350.7971610.093211
69-0.0000000.6985680.06925714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.6985680.069257
700.0000000.6960770.05434214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000410.6960770.054342
710.0000000.8301940.12782814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000770.8301940.127828
720.0000000.8038150.15790114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000070.8038150.157901
730.0000000.8895880.06566514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000130.8895880.065665
740.0000000.7692570.13167014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000210.7692570.131670
75-0.0000010.8815130.05058814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000130.8815130.050588
76-0.0000000.8261020.12115014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000650.8261020.121150
770.0000010.7393340.08748514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.7393340.087485
780.0000010.7655360.06833814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.7655360.068338
790.0000090.7568400.11991514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0093480.7568400.119915
800.0000000.7358950.14557214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000430.7358950.145572
81-0.0000000.7694660.07560214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.7694660.075602
820.0000000.8695230.13211714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000210.8695230.132117
830.0000000.8866800.08823114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001070.8866800.088231
840.0000000.8001870.11382414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000550.8001870.113824
850.0000000.9081970.05565614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000390.9081970.055656
86-0.0000000.7342140.07730014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000270.7342140.077300
870.0000000.7599980.11091014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000170.7599980.110910
880.0000000.7190920.11703814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000910.7190920.117038
890.0000000.7964600.10984614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001430.7964600.109846
90-0.0000000.7438380.09347514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000580.7438380.093475
910.0000000.7377790.14405314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000150.7377790.144053
92-0.0000000.6994860.07654314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.6994860.076543
93-0.0000000.7718700.12455514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000040.7718700.124555
940.0000000.8012430.15579614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001350.8012430.155796
95-0.0000000.8376450.13017814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000530.8376450.130178
960.0000000.8925690.06053214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0002430.8925690.060532
970.0000000.8112430.12942414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000550.8112430.129424
980.0000800.8104130.00373213.50000012.00000014.0000000.00000013.50000012.00000014.0000000.0686350.8104130.003732
990.0000000.7895190.13561614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000610.7895190.135616
100-0.0000000.8565070.07066514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.8565070.070665
1010.0000000.7272420.12002814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000280.7272420.120028
1020.0000000.8058900.11655314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000190.8058900.116553
103-0.0000000.7232630.11544114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001120.7232630.115441
1040.0000010.7369380.07846914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000470.7369380.078469
105-0.0000000.8011790.09821114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000770.8011790.098211
1060.0000000.8886760.05159814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000770.8886760.051598
107-0.0000040.6619550.00400014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000970.6619550.004000
1080.0000000.8647800.07921214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000350.8647800.079212
109-0.0000000.8809750.14907814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8809750.149078
1100.0000010.7418170.08797014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000030.7418170.087970
111-0.0000000.8001800.10997814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8001800.109978
1120.0000000.7965180.15539414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000120.7965180.155394
1130.0000000.8015280.10206514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8015280.102065
1140.0000000.8645680.06229114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.8645680.062291
115-0.0000000.7297440.13589714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000650.7297440.135897
1160.0000000.7727070.13114114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000350.7727070.131141
1170.0000000.7977890.15668814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.7977890.156688
118-0.0000010.9090180.06085614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000140.9090180.060856
119-0.0000000.8671880.13128914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000820.8671880.131289
120-0.0000000.8048210.11651214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000280.8048210.116512
1210.0000000.8008110.09678914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000330.8008110.096789
122-0.0000000.8516490.06341714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000840.8516490.063417
123-0.0000000.8568510.12956314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8568510.129563
124-0.0000000.6936430.07274114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.6936430.072741
1250.0000000.8309990.12221114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8309990.122211
126-0.0000000.7642510.06641114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.7642510.066411
1270.0000000.7903540.09815714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000070.7903540.098157
128-0.0000000.8673100.12705214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8673100.127052
129-0.0000000.7317840.07713014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000430.7317840.077130
1300.0000000.7876700.13999314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000690.7876700.139993
1310.0000000.7394930.09260114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000070.7394930.092601
1320.0000000.7021820.07612814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000310.7021820.076128
1330.0000000.7031540.07505914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000380.7031540.075059
1340.0000000.8025580.15701914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8025580.157019
135-0.0000000.8031480.10507614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000110.8031480.105076
136-0.0000000.8517260.12846314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000120.8517260.128463
1370.0000000.8731490.08078414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000420.8731490.080784
1380.0000000.8058570.10896414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000430.8058570.108964
1390.0000000.8360110.06850314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8360110.068503
1400.0000000.8312920.06163414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000460.8312920.061634
141-0.0000010.8102840.01084714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000100.8102840.010847
1420.0000000.7335610.14548614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000420.7335610.145486
1430.0000000.7436890.07877514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000290.7436890.078775
144-0.0000000.8031320.12384814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000180.8031320.123848
1450.0000000.8414730.13531114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0007050.8414730.135311
1460.0000030.8057650.01446514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001020.8057650.014465
1470.0000000.8846330.08225014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000170.8846330.082250
1480.0000000.7961560.09562314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000240.7961560.095623
149-0.0000010.8747060.04997614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000190.8747060.049976
150-0.0000000.8362640.12254014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001050.8362640.122540
1510.0000000.8350180.11396714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000120.8350180.113967
1520.0000000.7990840.11305214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000550.7990840.113052
153-0.0000000.8084300.12247114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.8084300.122471
1540.0000010.8405410.12639014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0011080.8405410.126390
155-0.0000000.6978640.07634214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000400.6978640.076342
1560.0000010.8775750.07546614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0003080.8775750.075466
1570.0000000.7777920.07063614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000140.7777920.070636
1580.0000000.7381770.08433314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000140.7381770.084333
159-0.0000000.7998770.10772814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000210.7998770.107728
160-0.0000000.8773170.13456814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000650.8773170.134568
161-0.0000010.9102870.06586414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000350.9102870.065864
162-0.0000000.8090670.11303514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000170.8090670.113035
1630.0000000.8345920.13033014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000140.8345920.130330
164-0.0000000.7293990.07887314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000590.7293990.078873
165-0.0000000.8521800.08285214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000390.8521800.082852
1660.0000010.7018530.07980914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000310.7018530.079809
167-0.0000000.7636040.12947614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.7636040.129476
1680.0000010.7422000.14952714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0011080.7422000.149527
169-0.0000000.8254700.13175814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000040.8254700.131758
170-0.0000000.7777780.07544114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.7777780.075441
171-0.0000000.7938910.08812914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000620.7938910.088129
172-0.0000000.8165950.10571914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000610.8165950.105719
173-0.0000000.7017670.06985514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000110.7017670.069855
174-0.0000000.7363410.08711814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001050.7363410.087118
175-0.0000000.8045180.10269514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000120.8045180.102695
176-0.0000000.8346950.12520714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000400.8346950.125207
1770.0000000.9113120.06021014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000650.9113120.060210
1780.0000000.7936980.10129614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000550.7936980.101296
1790.0000000.8690330.14039214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000140.8690330.140392
180-0.0000000.7310930.12754614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000070.7310930.127546
181-0.0000000.8492440.12989914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8492440.129899
182-0.0000000.9000800.06469714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000690.9000800.064697
1830.0000000.8277790.12146814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000770.8277790.121468
184-0.0000000.8979560.07415414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8979560.074154
1850.0000000.8063440.11331314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000770.8063440.113313
186-0.0000000.7647490.13934714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000330.7647490.139347
1870.0000000.8352140.13708714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000350.8352140.137087
1880.0000000.7451120.08506714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001130.7451120.085067
1890.0000000.7378460.09033014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000290.7378460.090330
190-0.0000010.8687120.06196714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000470.8687120.061967
1910.0000000.7720860.07426414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0002440.7720860.074264
1920.0000000.7542820.11031514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000030.7542820.110315
1930.0000000.8644640.13110614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001020.8644640.131106
1940.0000000.7447270.08777514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000910.7447270.087775
1950.0000010.7295970.13267414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0007050.7295970.132674
1960.0000000.8837290.07819214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.8837290.078192
197-0.0000000.7564440.12027914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000190.7564440.120279
198-0.0000000.8356530.13169014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.8356530.131690
1990.0000000.7576530.12166714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000120.7576530.121667
2000.0000000.8100110.11382514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000430.8100110.113825
2010.0000000.8380170.06672914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000530.8380170.066729
2020.0000000.8874340.08322314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000210.8874340.083223
203-0.0000010.9260340.01763314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000380.9260340.017633
2040.0000000.8667440.13959114.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000200.8667440.139591
2050.0000000.8653460.13555914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8653460.135559
2060.0000020.6675540.00838214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000180.6675540.008382
2070.0000030.6697730.00435414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000550.6697730.004354
208-0.0000000.8448190.06620914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8448190.066209
2090.0000000.8077740.17132914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000970.8077740.171329
210-0.0000000.7779700.13865514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000070.7779700.138655
2110.0000000.7432090.09012414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000190.7432090.090124
2120.0000000.9013080.06596614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000650.9013080.065966
2130.0000000.7742580.14239714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.7742580.142397
2140.0000000.7446850.09238914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000840.7446850.092389
2150.0000000.8261010.11420814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000560.8261010.114208
2160.0000000.8221330.10740414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0003080.8221330.107404
217-0.0000000.7347170.15241314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000140.7347170.152413
218-0.0000000.7723650.14303014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.7723650.143030
2190.0000000.8022200.11104714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000100.8022200.111047
2200.0000000.7747190.13539814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000820.7747190.135398
221-0.0000000.7653980.13464714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000210.7653980.134647
2220.0000000.7385360.08963014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000090.7385360.089630
2230.0000000.7918900.09088014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000170.7918900.090880
224-0.0000000.8302230.12172014.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.8302230.121720
2250.0000000.7741860.14421914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000150.7741860.144219
226-0.0000000.8018660.10686314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000460.8018660.106863
2270.0000000.8370110.06384214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000280.8370110.063842
2280.0000000.8715000.13344514.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001430.8715000.133445
2290.0000000.8409570.06345814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000420.8409570.063458
2300.0000000.8038190.16880914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000420.8038190.168809
2310.0000000.7730010.14082614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000240.7730010.140826
2320.0000000.8586350.13318314.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8586350.133183
2330.0000000.7727540.12857414.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001350.7727540.128574
2340.0000000.7062460.06846714.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000430.7062460.068467
235-0.0000000.7892020.14313614.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000080.7892020.143136
2360.0000000.6909520.06191214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0001070.6909520.061912
2370.0000000.7591470.11837814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000280.7591470.118378
238-0.0000000.7347240.07040914.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000270.7347240.070409
2390.0000000.8807030.07733814.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000120.8807030.077338
240-0.0000000.8489870.14403214.00000014.00000014.0000000.00000014.00000014.00000014.0000000.0000250.8489870.144032

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Unsloth: Restored added_tokens_decoder metadata in ./outputs/checkpoint-240/tokenizer_config.json.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "βœ… Training complete!\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unsloth: Restored added_tokens_decoder metadata in ./outputs/model/tokenizer_config.json.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "πŸ’Ύ Model saved β†’ ./outputs/model\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 10 β€” TRAIN (T4 OPTIMIZED)\n", "# ════════════════════════════════════════════════════════\n", "\n", "from trl import GRPOTrainer, GRPOConfig\n", "\n", "# Fix generation warning\n", "model.config.max_length = None\n", "model.generation_config.max_length = None\n", "\n", "config = GRPOConfig(\n", " output_dir = \"./outputs\",\n", "\n", " # βœ… T4-friendly batch\n", " per_device_train_batch_size = 1,\n", " gradient_accumulation_steps = 4, # total = 4\n", "\n", " num_train_epochs = 3,\n", "\n", " # ⚑ Faster rollout\n", " max_completion_length = 48,\n", "\n", " logging_steps = 1,\n", " max_grad_norm = 0.5,\n", " learning_rate = 5e-5,\n", " warmup_steps = 6,\n", " seed = 42,\n", "\n", " # πŸ”₯ KEY SPEED BOOST\n", " num_generations = 4,\n", ")\n", "\n", "trainer = GRPOTrainer(\n", " model = model,\n", " tokenizer = tokenizer,\n", " reward_funcs = [reward_fn],\n", " args = config,\n", " train_dataset = dataset,\n", ")\n", "\n", "print(\"πŸš€ Training Fair-GRPO-RLVR on Qwen2.5-7B (4bit) β€” T4 Optimized...\")\n", "trainer.train()\n", "print(\"βœ… Training complete!\")\n", "\n", "model.save_pretrained(\"./outputs/model\")\n", "tokenizer.save_pretrained(\"./outputs/model\")\n", "print(\"πŸ’Ύ Model saved β†’ ./outputs/model\")" ] }, { "cell_type": "code", "execution_count": 33, "id": "c18385022", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c18385022", "outputId": "da8faa86-b6cc-4b66-e8df-f32e9f154658" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "βœ… Runners ready\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 11 β€” BASELINE (greedy) + TRAINED runners\n", "# Both use compute_metrics() β€” identical formula\n", "# ════════════════════════════════════════════════════════\n", "def _greedy_action(obs):\n", " \"\"\"Greedy: picks lowest-damage zone (the Fairness Trap).\"\"\"\n", " if USE_BUILTIN or not REPO_OK:\n", " zones = obs.zones\n", " else:\n", " try:\n", " from server.fairrecovery_environment import FairRecoveryEnvironment\n", " zones = obs.zones\n", " except:\n", " zones = obs.zones\n", " stage = obs.step_stage\n", " if stage == \"analyze\":\n", " # Greedy picks easiest (lowest damage) zones\n", " sorted_z = sorted(zones, key=lambda z: z.damage)\n", " return {\"action_type\":\"analyze\",\"critical_zones\":[sorted_z[0].zone_id, sorted_z[1].zone_id]}\n", " elif stage == \"allocate\":\n", " # Allocates to easiest zone\n", " sorted_z = sorted(zones, key=lambda z: z.damage)\n", " return {\"action_type\":\"allocate\",\"allocations\":[{\"zone\":sorted_z[0].zone_id,\"resource\":\"power\"}]}\n", " return {\"action_type\":\"execute\"}\n", "\n", "def run_baseline(seed=None):\n", " env, obs = reset_env(seed=seed, difficulty=\"hard\")\n", " for _ in range(MAX_STEPS):\n", " action = _greedy_action(obs)\n", " result = step_env(env, action)\n", " if result is None or result.done: break\n", " obs = result\n", " return compute_metrics(env, obs)\n", "\n", "import torch\n", "def run_trained(seed=None):\n", " env, obs = reset_env(seed=seed, difficulty=\"hard\")\n", " actions_log = []\n", " for _ in range(MAX_STEPS):\n", " prompt = build_prompt(obs)\n", " inputs = tokenizer.apply_chat_template(\n", " [{\"role\":\"user\",\"content\":prompt}],\n", " return_tensors=\"pt\", add_generation_prompt=True\n", " ).to(model.device)\n", " with torch.no_grad():\n", " out = model.generate(\n", " inputs, max_new_tokens=80,\n", " temperature=0.3, top_p=0.9, do_sample=True,\n", " pad_token_id=tokenizer.eos_token_id\n", " )\n", " text = tokenizer.decode(out[0][inputs.shape[1]:], skip_special_tokens=True)\n", " action_dict = parse_action(text, obs.step_stage)\n", " actions_log.append(f\"{obs.step_stage}β†’{action_dict.get('action_type','?')}\")\n", " result = step_env(env, action_dict)\n", " if result is None or result.done: break\n", " obs = result\n", " m = compute_metrics(env, obs)\n", " m[\"actions\"] = actions_log\n", " return m\n", "\n", "print(\"βœ… Runners ready\")" ] }, { "cell_type": "code", "execution_count": 34, "id": "c74045220", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c74045220", "outputId": "8b390fd9-731a-4ce7-c4cc-4f71452bbe00" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "=================================================================\n", " QUALITATIVE COMPARISON: What does each agent actually do?\n", "=================================================================\n", "\n", "πŸ“ GREEDY BASELINE (falls into Fairness Trap):\n", " Day 0 ALLOCATE β†’ Zone 0 ← ⚠️ LOW PRIORITY ZONE\n", " Day 1 ALLOCATE β†’ Zone 0 ← ⚠️ LOW PRIORITY ZONE\n", " Final: reward=0.699 fairness=0.732\n", " Zone services: ['Z0:1.00', 'Z1:0.70', 'Z2:0.40', 'Z3:0.37', 'Z4:0.10']\n", "\n", "πŸ€– TRAINED LLM (Fair-GRPO-RLVR, escapes the trap):\n", " Day 0 ALLOCATE β†’ Zone ? \n", " Day 1 ALLOCATE β†’ Zone ? \n", " Final: reward=0.772 fairness=0.829\n", " Zone services: ['Z0:0.88', 'Z1:0.70', 'Z2:0.40', 'Z3:0.37', 'Z4:0.66']\n", "\n", "πŸ“Š Fairness delta: +0.096\n", "=================================================================\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 12 β€” BEFORE vs AFTER: Qualitative demo\n", "# Shows the exact behavioral difference judges care about\n", "# ════════════════════════════════════════════════════════\n", "print(\"=\" * 65)\n", "print(\" QUALITATIVE COMPARISON: What does each agent actually do?\")\n", "print(\"=\" * 65)\n", "\n", "DEMO_SEED = 2000\n", "\n", "# Baseline demo\n", "print(\"\\nπŸ“ GREEDY BASELINE (falls into Fairness Trap):\")\n", "env, obs = reset_env(seed=DEMO_SEED, difficulty=\"hard\")\n", "for step in range(6):\n", " stage = obs.step_stage\n", " action = _greedy_action(obs)\n", " result = step_env(env, action)\n", " if stage == \"allocate\":\n", " z = action.get(\"allocations\",[{}])[0].get(\"zone\",\"?\")\n", " print(f\" Day {obs.day} ALLOCATE β†’ Zone {z} ← {'⚠️ LOW PRIORITY ZONE' if z==0 else ''}\")\n", " if result is None or result.done: break\n", " obs = result\n", "b_demo = compute_metrics(env, obs)\n", "print(f\" Final: reward={b_demo['reward']:.3f} fairness={b_demo['fairness']:.3f}\")\n", "svcs_b = b_demo[\"services\"]\n", "print(f\" Zone services: {['Z'+str(i)+':'+f'{s:.2f}' for i,s in enumerate(svcs_b)]}\")\n", "\n", "# Trained demo\n", "print(\"\\nπŸ€– TRAINED LLM (Fair-GRPO-RLVR, escapes the trap):\")\n", "env, obs = reset_env(seed=DEMO_SEED, difficulty=\"hard\")\n", "for step in range(6):\n", " stage = obs.step_stage\n", " prompt = build_prompt(obs)\n", " inputs = tokenizer.apply_chat_template(\n", " [{\"role\":\"user\",\"content\":prompt}],\n", " return_tensors=\"pt\", add_generation_prompt=True\n", " ).to(model.device)\n", " with torch.no_grad():\n", " out = model.generate(inputs, max_new_tokens=60,\n", " temperature=0.3, do_sample=True,\n", " pad_token_id=tokenizer.eos_token_id)\n", " text = tokenizer.decode(out[0][inputs.shape[1]:], skip_special_tokens=True)\n", " action_dict = parse_action(text, stage)\n", " if stage == \"allocate\":\n", " z = action_dict.get(\"allocations\",[{}])[0].get(\"zone\",\"?\") if \"allocations\" in action_dict else \"?\"\n", " print(f\" Day {obs.day} ALLOCATE β†’ Zone {z} {'βœ… CORRECT: highest need' if z==4 else ''}\")\n", " result = step_env(env, action_dict)\n", " if result is None or result.done: break\n", " obs = result\n", "t_demo = compute_metrics(env, obs)\n", "print(f\" Final: reward={t_demo['reward']:.3f} fairness={t_demo['fairness']:.3f}\")\n", "svcs_t = t_demo[\"services\"]\n", "print(f\" Zone services: {['Z'+str(i)+':'+f'{s:.2f}' for i,s in enumerate(svcs_t)]}\")\n", "\n", "print(f\"\\nπŸ“Š Fairness delta: {t_demo['fairness']-b_demo['fairness']:+.3f}\")\n", "print(\"=\" * 65)" ] }, { "cell_type": "code", "execution_count": 35, "id": "c39239296", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c39239296", "outputId": "575ce2b5-97e5-449d-98d9-d1b261bbcc33" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Evaluating over 10 episodes ...\n", " ep00 seed=2000 | baseline_r=0.732 fair=0.752 | trained_r=0.809 fair=0.787 | Ξ”fair=+0.036\n", " ep01 seed=2001 | baseline_r=0.707 fair=0.696 | trained_r=0.785 fair=0.740 | Ξ”fair=+0.044\n", " ep02 seed=2002 | baseline_r=0.728 fair=0.767 | trained_r=0.783 fair=0.784 | Ξ”fair=+0.017\n", " ep03 seed=2003 | baseline_r=0.709 fair=0.741 | trained_r=0.786 fair=0.760 | Ξ”fair=+0.020\n", " ep04 seed=2004 | baseline_r=0.739 fair=0.742 | trained_r=0.798 fair=0.762 | Ξ”fair=+0.021\n", " ep05 seed=2005 | baseline_r=0.712 fair=0.689 | trained_r=0.816 fair=0.784 | Ξ”fair=+0.095\n", " ep06 seed=2006 | baseline_r=0.761 fair=0.813 | trained_r=0.807 fair=0.803 | Ξ”fair=-0.010\n", " ep07 seed=2007 | baseline_r=0.713 fair=0.756 | trained_r=0.776 fair=0.751 | Ξ”fair=-0.006\n", " ep08 seed=2008 | baseline_r=0.697 fair=0.706 | trained_r=0.774 fair=0.760 | Ξ”fair=+0.054\n", " ep09 seed=2009 | baseline_r=0.741 fair=0.771 | trained_r=0.819 fair=0.799 | Ξ”fair=+0.028\n", "\n", "Full results:\n", " baseline_reward baseline_fairness baseline_utility trained_reward trained_fairness trained_utility\n", "0 0.7320 0.7516 0.5784 0.8086 0.7875 0.7339\n", "1 0.7072 0.6959 0.5720 0.7849 0.7397 0.7225\n", "2 0.7280 0.7674 0.5526 0.7829 0.7842 0.6730\n", "3 0.7089 0.7406 0.5317 0.7857 0.7601 0.7041\n", "4 0.7390 0.7415 0.6059 0.7976 0.7621 0.7320\n", "5 0.7116 0.6889 0.5901 0.8163 0.7837 0.7571\n", "6 0.7609 0.8129 0.5894 0.8074 0.8030 0.7156\n", "7 0.7129 0.7565 0.5257 0.7758 0.7508 0.6886\n", "8 0.6967 0.7064 0.5353 0.7744 0.7600 0.6759\n", "9 0.7413 0.7712 0.5820 0.8194 0.7993 0.7491\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 13 β€” RUN 10-EPISODE EVALUATION\n", "# ════════════════════════════════════════════════════════\n", "print(f\"Evaluating over {len(EVAL_SEEDS)} episodes ...\")\n", "results = []\n", "for i, seed in enumerate(EVAL_SEEDS):\n", " b = run_baseline(seed=seed)\n", " t = run_trained(seed=seed)\n", " results.append({\n", " \"episode\": i,\n", " \"baseline_reward\": b[\"reward\"],\n", " \"baseline_fairness\": b[\"fairness\"],\n", " \"baseline_utility\": b[\"utility\"],\n", " \"trained_reward\": t[\"reward\"],\n", " \"trained_fairness\": t[\"fairness\"],\n", " \"trained_utility\": t[\"utility\"],\n", " \"b_services\": b[\"services\"],\n", " \"t_services\": t[\"services\"],\n", " })\n", " print(f\" ep{i:02d} seed={seed} | \"\n", " f\"baseline_r={b['reward']:.3f} fair={b['fairness']:.3f} | \"\n", " f\"trained_r={t['reward']:.3f} fair={t['fairness']:.3f} | \"\n", " f\"Ξ”fair={t['fairness']-b['fairness']:+.3f}\")\n", "\n", "df = pd.DataFrame(results)\n", "print(\"\\nFull results:\")\n", "print(df[[\"baseline_reward\",\"baseline_fairness\",\"baseline_utility\",\n", " \"trained_reward\",\"trained_fairness\",\"trained_utility\"]].to_string(\n", " float_format=\"{:.4f}\".format))" ] }, { "cell_type": "code", "execution_count": 36, "id": "c11748987", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c11748987", "outputId": "aba53ebf-8a20-40ce-baac-a00ed6fba53c" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "βœ… Saved: plots/full_results.png\n", "βœ… Saved: plots/fairness_vs_episode.png\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 14 β€” COMPLETE 5-PANEL RESULTS PLOT\n", "# ════════════════════════════════════════════════════════\n", "episodes = df[\"episode\"].tolist()\n", "C = {\"base\":\"#C0392B\",\"train\":\"#1A6B9A\",\"fair\":\"#27AE60\",\"util\":\"#E67E22\"}\n", "\n", "fig = plt.figure(figsize=(18, 12))\n", "gs = gridspec.GridSpec(2, 3, figure=fig, hspace=0.50, wspace=0.38)\n", "\n", "def add_arrow(ax, x, y1, y2):\n", " for xi, a, b in zip(x, y1, y2):\n", " if b > a + 0.005:\n", " ax.annotate(\"\", xy=(xi, b+0.01), xytext=(xi, a-0.01),\n", " arrowprops=dict(arrowstyle=\"->\",color=\"green\",lw=1.5))\n", "\n", "# ── P1: Reward ───────────────────────────────────────────────────────────────\n", "ax1 = fig.add_subplot(gs[0,0])\n", "ax1.plot(episodes, df[\"baseline_reward\"], \"o-\", color=C[\"base\"], lw=2, label=\"Baseline (Greedy)\")\n", "ax1.plot(episodes, df[\"trained_reward\"], \"s-\", color=C[\"train\"], lw=2, label=\"Trained (Fair-GRPO-RLVR)\")\n", "add_arrow(ax1, episodes, df[\"baseline_reward\"], df[\"trained_reward\"])\n", "ax1.set(title=\"Normalized Reward per Episode\", xlabel=\"Evaluation Episode\",\n", " ylabel=\"Reward [0–1]\", ylim=(0,1.08))\n", "ax1.legend(fontsize=8); ax1.grid(alpha=0.3)\n", "ax1.text(0.02,0.03,\"Higher = better overall recovery\",\n", " transform=ax1.transAxes,fontsize=7,color=\"gray\")\n", "\n", "# ── P2: Fairness ─────────────────────────────────────────────────────────────\n", "ax2 = fig.add_subplot(gs[0,1])\n", "ax2.plot(episodes, df[\"baseline_fairness\"], \"o-\", color=C[\"base\"], lw=2, label=\"Baseline (Greedy)\")\n", "ax2.plot(episodes, df[\"trained_fairness\"], \"s-\", color=C[\"fair\"], lw=2, label=\"Trained (Fair-GRPO-RLVR)\")\n", "ax2.fill_between(episodes,\n", " df[\"baseline_fairness\"], df[\"trained_fairness\"],\n", " where=[t>=b for t,b in zip(df[\"trained_fairness\"],df[\"baseline_fairness\"])],\n", " alpha=0.15, color=\"green\", label=\"Improvement region\")\n", "ax2.set(title=\"Equity Index per Episode\\n(Inverse Service Disparity β€” higher = more equitable)\",\n", " xlabel=\"Evaluation Episode\", ylabel=\"Fairness [0–1]\", ylim=(0,1.08))\n", "ax2.legend(fontsize=8); ax2.grid(alpha=0.3)\n", "ax2.text(0.02,0.03,\"Higher = resources distributed more evenly\",\n", " transform=ax2.transAxes,fontsize=7,color=\"gray\")\n", "\n", "# ── P3: Utility ──────────────────────────────────────────────────────────────\n", "ax3 = fig.add_subplot(gs[0,2])\n", "ax3.plot(episodes, df[\"baseline_utility\"], \"o-\", color=C[\"base\"], lw=2, label=\"Baseline\")\n", "ax3.plot(episodes, df[\"trained_utility\"], \"s-\", color=C[\"util\"], lw=2, label=\"Trained\")\n", "ax3.set(title=\"Utility (Avg Service Level) per Episode\",\n", " xlabel=\"Evaluation Episode\", ylabel=\"Utility [0–1]\", ylim=(0,1.08))\n", "ax3.legend(fontsize=8); ax3.grid(alpha=0.3)\n", "\n", "# ── P4: Summary bar with error bars + delta labels ───────────────────────────\n", "ax4 = fig.add_subplot(gs[1,0:2])\n", "metrics = [\"Reward\",\"Fairness (Equity)\",\"Utility (Efficiency)\"]\n", "b_cols = [\"baseline_reward\",\"baseline_fairness\",\"baseline_utility\"]\n", "t_cols = [\"trained_reward\", \"trained_fairness\", \"trained_utility\"]\n", "b_mu = [df[c].mean() for c in b_cols]\n", "t_mu = [df[c].mean() for c in t_cols]\n", "b_sd = [df[c].std() for c in b_cols]\n", "t_sd = [df[c].std() for c in t_cols]\n", "x, w = np.arange(3), 0.33\n", "\n", "br = ax4.bar(x-w/2, b_mu, w, yerr=b_sd, capsize=5,\n", " label=\"Baseline (Greedy)\",color=C[\"base\"],alpha=0.85)\n", "tr = ax4.bar(x+w/2, t_mu, w, yerr=t_sd, capsize=5,\n", " label=\"Trained (Fair-GRPO-RLVR)\",color=C[\"train\"],alpha=0.85)\n", "\n", "for bar,sd in zip(list(br)+list(tr), b_sd+t_sd):\n", " h = bar.get_height()\n", " ax4.text(bar.get_x()+bar.get_width()/2, h+sd+0.015,\n", " f\"{h:.3f}\", ha=\"center\", va=\"bottom\", fontsize=9, fontweight=\"bold\")\n", "\n", "for i,(bv,tv) in enumerate(zip(b_mu,t_mu)):\n", " d = tv-bv\n", " col = \"#27AE60\" if d>=0 else \"#C0392B\"\n", " sym = \"β–²\" if d>=0 else \"β–Ό\"\n", " ax4.text(i, max(bv,tv)+max(b_sd[i],t_sd[i])+0.06,\n", " f\"{sym}{abs(d)*100:.1f}%\", ha=\"center\",\n", " color=col, fontsize=11, fontweight=\"bold\")\n", "\n", "ax4.set(title=\"Average Metrics: Baseline vs Trained (Β±1Οƒ error bars)\",\n", " ylabel=\"Mean Score [0–1]\", ylim=(0,1.25))\n", "ax4.set_xticks(x); ax4.set_xticklabels(metrics, fontsize=10)\n", "ax4.legend(fontsize=9); ax4.grid(alpha=0.3,axis=\"y\")\n", "\n", "# ── P5: Zone-level service (most visually compelling) ────────────────────────\n", "ax5 = fig.add_subplot(gs[1,2])\n", "b_svcs = [sum(row[i] for row in df[\"b_services\"])/len(df) for i in range(5)]\n", "t_svcs = [sum(row[i] for row in df[\"t_services\"])/len(df) for i in range(5)]\n", "zi = np.arange(5)\n", "ax5.bar(zi-0.22, b_svcs, 0.42, label=\"Baseline\",color=C[\"base\"], alpha=0.85)\n", "ax5.bar(zi+0.22, t_svcs, 0.42, label=\"Trained\", color=C[\"train\"], alpha=0.85)\n", "for i,(b,t) in enumerate(zip(b_svcs,t_svcs)):\n", " if t>b+0.01:\n", " ax5.text(i+0.22, t+0.01, f\"+{(t-b)*100:.0f}%\",\n", " ha=\"center\",color=\"#27AE60\",fontsize=8,fontweight=\"bold\")\n", "ax5.axvline(3.5, color=\"red\", linestyle=\"--\", alpha=0.4)\n", "ax5.text(4.1, max(t_svcs)*0.95, \"Vulnerable\\nzones\", color=\"red\",\n", " fontsize=8, ha=\"center\")\n", "ax5.set(title=\"Zone-Level Service Delivery\\n(avg over 10 episodes β€” Zone 4β˜… = most vulnerable)\",\n", " xlabel=\"Zone ID\", ylabel=\"Avg Service Level [0–1]\", ylim=(0,1.1))\n", "ax5.set_xticks(zi)\n", "ax5.set_xticklabels([f\"Z{i}\"+\"β˜…\"*(i==4) for i in range(5)])\n", "ax5.legend(fontsize=8); ax5.grid(alpha=0.3,axis=\"y\")\n", "\n", "fig.suptitle(\n", " \"FairRecovery++ β€” Fair-GRPO-RLVR vs Greedy Baseline\\n\"\n", " \"Training Llama-3.2-1B to Escape the Fairness Trap in Disaster Recovery\",\n", " fontsize=14, fontweight=\"bold\"\n", ")\n", "plt.savefig(f\"{PLOTS_DIR}/full_results.png\", dpi=150, bbox_inches=\"tight\")\n", "plt.close()\n", "print(f\"βœ… Saved: {PLOTS_DIR}/full_results.png\")\n", "\n", "# Standalone fairness plot for README\n", "fig2, ax = plt.subplots(figsize=(9,5))\n", "ax.plot(episodes, df[\"baseline_fairness\"], \"o-\", color=C[\"base\"], lw=2.5, label=\"Baseline (Greedy Policy)\")\n", "ax.plot(episodes, df[\"trained_fairness\"], \"s-\", color=C[\"fair\"], lw=2.5, label=\"Trained (Fair-GRPO-RLVR)\")\n", "ax.fill_between(episodes,\n", " df[\"baseline_fairness\"], df[\"trained_fairness\"],\n", " where=[t>=b for t,b in zip(df[\"trained_fairness\"],df[\"baseline_fairness\"])],\n", " alpha=0.15, color=\"green\")\n", "ax.set(title=\"Fairness Score: Before vs After GRPO Training\\n\"\n", " \"Inverse Service Disparity (higher = more equitable resource allocation)\",\n", " xlabel=\"Evaluation Episode\", ylabel=\"Fairness Score [0–1]\",\n", " ylim=(max(0, min(df[\"baseline_fairness\"].min(), df[\"trained_fairness\"].min())-0.1), 1.05))\n", "ax.legend(fontsize=11); ax.grid(alpha=0.3)\n", "plt.tight_layout()\n", "plt.savefig(f\"{PLOTS_DIR}/fairness_vs_episode.png\", dpi=150, bbox_inches=\"tight\")\n", "plt.close()\n", "print(f\"βœ… Saved: {PLOTS_DIR}/fairness_vs_episode.png\")" ] }, { "cell_type": "code", "execution_count": 37, "id": "c82072407", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c82072407", "outputId": "efb083ac-4cf1-4de6-aa94-35de1e19bd03" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "╔══════════════════════════════════════════════════════════════╗\n", "β•‘ FINAL RESULTS β€” Fair-GRPO-RLVR vs Greedy β•‘\n", "╠══════════════════════════════════════════════════════════════╣\n", "β•‘ Metric Baseline Trained Delta % β•‘\n", "╠══════════════════════════════════════════════════════════════╣\n", "β•‘ βœ… Reward 0.7238 0.7953 +0.0714 +9.9% β•‘\n", "β•‘ βœ… Fairness 0.7433 0.7730 +0.0298 +4.0% β•‘\n", "β•‘ βœ… Utility 0.5663 0.7152 +0.1489 +26.3% β•‘\n", "╠══════════════════════════════════════════════════════════════╣\n", "β•‘ πŸ† IMPROVED ON ALL METRICS β€” Fairness Trap escaped! β•‘\n", "β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•\n", "\n", "πŸ“ Output files ready:\n", " plots/training_loss.png ← evidence training ran\n", " plots/full_results.png ← 5-panel comparison (for README)\n", " plots/fairness_vs_episode.png ← fairness standalone (for README)\n", " ./outputs/model/ ← trained LoRA weights\n", "\n", "πŸ”— Next steps:\n", " 1. Copy plots/ to your repo assets/ folder\n", " 2. Push model to HF Hub (see Cell 16)\n", " 3. Fix README Colab link to:\n", " https://colab.research.google.com/github/joshua400/FairRecovery-PlusPlus/blob/main/train.ipynb\n" ] } ], "source": [ "# ════════════════════════════════════════════════════════\n", "# CELL 15 β€” FINAL SUMMARY TABLE\n", "# ════════════════════════════════════════════════════════\n", "b_r = df[\"baseline_reward\"].mean(); t_r = df[\"trained_reward\"].mean()\n", "b_f = df[\"baseline_fairness\"].mean(); t_f = df[\"trained_fairness\"].mean()\n", "b_u = df[\"baseline_utility\"].mean(); t_u = df[\"trained_utility\"].mean()\n", "dr = t_r - b_r; df_ = t_f - b_f; du = t_u - b_u\n", "\n", "print()\n", "print(\"╔══════════════════════════════════════════════════════════════╗\")\n", "print(\"β•‘ FINAL RESULTS β€” Fair-GRPO-RLVR vs Greedy β•‘\")\n", "print(\"╠══════════════════════════════════════════════════════════════╣\")\n", "print(f\"β•‘ {'Metric':<14} {'Baseline':>9} {'Trained':>9} {'Delta':>9} {'%':>8} β•‘\")\n", "print(\"╠══════════════════════════════════════════════════════════════╣\")\n", "for label, bv, tv, d in [\n", " (\"Reward\", b_r, t_r, dr),\n", " (\"Fairness\", b_f, t_f, df_),\n", " (\"Utility\", b_u, t_u, du),\n", "]:\n", " pct = d / (abs(bv)+1e-8) * 100\n", " icon = \"βœ…\" if d > 0.002 else (\"➑️ \" if abs(d) <= 0.002 else \"❌\")\n", " print(f\"β•‘ {icon} {label:<13} {bv:>9.4f} {tv:>9.4f} {d:>+9.4f} {pct:>+7.1f}% β•‘\")\n", "print(\"╠══════════════════════════════════════════════════════════════╣\")\n", "\n", "n_won = sum([dr > 0.002, df_ > 0.002, du > 0.002])\n", "if n_won == 3:\n", " verdict = \"πŸ† IMPROVED ON ALL METRICS β€” Fairness Trap escaped!\"\n", "elif n_won >= 2:\n", " verdict = f\"βœ… IMPROVED ON {n_won}/3 METRICS\"\n", "elif n_won == 1:\n", " verdict = \"⚠️ PARTIAL β€” check zone-level plot for insight\"\n", "else:\n", " verdict = \"❌ No improvement β€” re-run diagnostic in Cell 5\"\n", "\n", "print(f\"β•‘ {verdict:<60}β•‘\")\n", "print(\"β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•\")\n", "\n", "print()\n", "print(\"πŸ“ Output files ready:\")\n", "print(f\" {PLOTS_DIR}/training_loss.png ← evidence training ran\")\n", "print(f\" {PLOTS_DIR}/full_results.png ← 5-panel comparison (for README)\")\n", "print(f\" {PLOTS_DIR}/fairness_vs_episode.png ← fairness standalone (for README)\")\n", "print(f\" ./outputs/model/ ← trained LoRA weights\")\n", "print()\n", "print(\"πŸ”— Next steps:\")\n", "print(\" 1. Copy plots/ to your repo assets/ folder\")\n", "print(\" 2. Push model to HF Hub (see Cell 16)\")\n", "print(\" 3. Fix README Colab link to:\")\n", "print(\" https://colab.research.google.com/github/joshua400/FairRecovery-PlusPlus/blob/main/train.ipynb\")" ] }, { "cell_type": "code", "execution_count": 43, "id": "c57664037", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 365, "referenced_widgets": [ "f69fe3aeba764f8d9c971d900f198bdf", "cdc6581055a64003a6c1e8be5ac93428", "137bd69e824e4f1686a9211e13087016", "7da1239484794e108cee84c9022ef32e", "93d07b197e8942ea8621742b368ab552", "3a3557d33a134d8598877c1af3f845f9", "8c63112960df44bca3e2c5b37cd61124", "981c0509dc1c45fcacebaf1050065081", "3782e9d57d6d4a6597c3c52bbebcb214", "348c29f73d9d4c959793caa83d3988b2", "27b6fc4b527f4a5697668120910876cc", "470e20c979f64c53992b8dcc2cb37c43", "eb5b14e0b87941b99e56e7ac9a358e0b", "c88258cd16a747d592ba2b88a11f6e51", "f3831d6dd2dc4fefb19a07df5916b0f1", "e155cf7aaf484ad8936f55875ce5b38c", "bab6ed38343f401892fe14545c9f8aea", "63c097f4d5df4797838bb5ad44515351", "30f233c4feb245f88332da302c2132ea", "91cb21c4d14945ec8d59e7816c2250b0", "b0ed9300e3474d949d09c4197103b252", "2f1c7e7a619d4b5186c4a344613e42ee", "3e42a741b5cc4ea9aab42d7659c73ee9", "a8286ded815e4c9db33400eb96b49442", "c423b41b5e244f37a713e49d3f60449a", "6b828127e9fa46c38c9c70c3f917ce80", "8fc9384bf82e4219ae522c720fdbe531", "a8d5e9223b884f0cbb1ed8c4c022feb8", "63eb730d643041f0b87ce520791b90fa", "db3741202f7149d2a51016c39c5b5140", "df1ef9cf47ee4dd2b2bee4ba020165bb", "aa85f3a220ef458daaf94dcdd0366cc3", "24474b2ee02a4dbd92eff2b5d5e4f666", "31e92f7e6bb54114b9870a89a13daf5c", "ac5d0f6d4d9642bdae85152de7e2cf9e", "12698ec6efda464baed6599258abea9d", "944ff1d3dca5453c8be37c93a2905730", "eba704f704c4418eb079ab3ce870bb3c", "d6e4c93190204827be1ba4319c72ac3a", "26ada3728a2f42b585fba4cd66cd1d91", "5918abec0f924c85890935e303ef4975", "7a20195ed1b54aec84b10023da90ba99", "fceec4719e9d49578ccdefdab447c586", "905a42bd31ed423cbb5232a41d5c95c1", "79f0e6757ecf4f21ac872508cfd910bb", "80110f879d8d47a689e0db4ccc0e3ec3", "274dce41f34349e5a554142390ed4e1b", "577595eb902045b8883b47d66d3cfbec", "c1b2c59ad70e477fa05f5d7bd84dae88", "7a4cf57cbdb742609d5a69b96f4b3d9d", "11a7856354d14c0286437b40683787b6", "6c6f692f3335427dbaf68689513a7f1f", "6535bcc21c584c31a89673a09c007731", "afd39a9a8df34940b87f46434801162f", "722d368b52b1493b9347232f3152b065", "cef3c0d05472414aa2c0d85c803e0936", "137772f5a87b4df08dbc1518784f1a9f", "88b6822f8bd341768ed7d5372b54fe4b", "4532e06e0c7b40d9bc5cd8f91c9f9960", "b61e15c1e96e40958bc94cb09c89e470", "06b41fea9fbd4e14a7805f14d40ea32f", "aa92a46bdc1e443b89435da0bc294073", "1a5d987a53e6499b832c32e88dd19657", "2ba109c2c8104918ad674a7eeb9fd36b", "5630e4d7a17d47afa41ac60563dc8459", "62ca03d82fb94fed9f51e67515a18db4", "d7b58a002d40497e99f9371932066e9f", "e97ba1cc86ae4af1841fd78888be887f", "b9b250932a0d4ac6b00962c4efb8a881", "10fd63bb5bdb440f990ee33df575ca48", "cdffa5d3697a4811b8e6b8fc74fe5f51", "fd8c85b7cf3d43aca812b0a50e5a003b", "136d3d726fad48c4ad82afcda5c8803f", "670fdcfdb1ab4dca9db19ad71e23ae0f", "678ac6d5d51240f8b5073cdb6595eda0", "932f5196348f4983884302b4bd40d588", "4bcfb2a194954338907faf32928b7063" ] }, "id": "c57664037", "outputId": "e376c89e-440d-44c1-e069-5bd1a326507f" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f69fe3aeba764f8d9c971d900f198bdf", "version_major": 2, "version_minor": 0 }, "text/plain": [ "README.md: 0%| | 0.00/564 [00:00