Upload cipher-simpo-train.ipynb with huggingface_hub

cipher-simpo-train.ipynb

@@ -0,0 +1,238 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Cipher Stage 2 — SimPO Anti-Slop Training (Self-Contained)\n",
+    "\n",
+    "Pulls SFT-merged model from HF, generates preference pairs, trains SimPO, pushes result.\n",
+    "\n",
+    "**Runtime needed:** A100 (Colab Pro+). ~45-90 min total.\n",
+    "**Output:** `Auroraventures/cipher-simpo-merged` on HuggingFace.\n",
+    "\n",
+    "Best-in-class params (paper-optimal): beta=10, gamma=2.5, grad_accum=128"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": "# Cell 1 — Install\n!pip install -U unsloth trl datasets huggingface_hub -q\n!pip install --no-deps git+https://github.com/unslothai/unsloth.git@nightly -q\nimport os\nfrom google.colab import userdata\ntry:\n    os.environ['HF_TOKEN'] = userdata.get('HF_TOKEN')\nexcept Exception:\n    raise RuntimeError('Add HF_TOKEN to Colab Secrets (left sidebar key icon).')\nos.environ['WANDB_DISABLED'] = 'true'\nfrom huggingface_hub import login\nlogin(token=os.environ['HF_TOKEN'])\nprint('Setup done')"
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Cell 2 — Load SFT-merged model from HF (~5 min, 62GB)\n",
+    "import torch\n",
+    "from unsloth import FastLanguageModel\n",
+    "\n",
+    "MODEL_ID = 'Auroraventures/cipher-sft-merged'\n",
+    "MAX_SEQ = 8192\n",
+    "\n",
+    "model, tokenizer = FastLanguageModel.from_pretrained(\n",
+    "    model_name=MODEL_ID,\n",
+    "    max_seq_length=MAX_SEQ,\n",
+    "    load_in_4bit=True,\n",
+    "    dtype=None,\n",
+    ")\n",
+    "print(f'Loaded: VRAM {torch.cuda.max_memory_allocated()/1e9:.1f}GB')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Cell 3 — Apply fresh LoRA for SimPO\n",
+    "model = FastLanguageModel.get_peft_model(\n",
+    "    model,\n",
+    "    r=64,                  # paper-optimal for 31B with rsLoRA\n",
+    "    lora_alpha=64,\n",
+    "    target_modules=['q_proj','k_proj','v_proj','o_proj','gate_proj','up_proj','down_proj'],\n",
+    "    use_gradient_checkpointing='unsloth',\n",
+    "    use_rslora=True,       # critical for r=64 stability\n",
+    "    random_state=42,\n",
+    ")\n",
+    "model.print_trainable_parameters()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Cell 4 — Pull training data + slop tools from kr8tiv-training\n",
+    "import subprocess, sys\n",
+    "subprocess.run('rm -rf /content/kr8tiv-training && git clone --depth 1 https://github.com/kr8tiv-ai/kr8tiv-training.git /content/kr8tiv-training', shell=True, check=True)\n",
+    "sys.path.insert(0, '/content/kr8tiv-training')\n",
+    "from scripts.slop_detector import slop_score\n",
+    "from scripts.rejected_generator import generate_rejected\n",
+    "print('Tools loaded')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Cell 5 — Generate SimPO preference pairs from existing SFT data\n",
+    "import json\n",
+    "from pathlib import Path\n",
+    "\n",
+    "SFT_DATA = '/content/kr8tiv-training/data/prompts/awwwards-sft.jsonl'\n",
+    "PAIRS_OUT = '/content/simpo_pairs.jsonl'\n",
+    "\n",
+    "examples = []\n",
+    "with open(SFT_DATA) as f:\n",
+    "    for line in f:\n",
+    "        if line.strip():\n",
+    "            try: examples.append(json.loads(line))\n",
+    "            except: pass\n",
+    "\n",
+    "print(f'Loaded {len(examples)} SFT examples')\n",
+    "\n",
+    "# Build preference pairs\n",
+    "pairs = []\n",
+    "skipped = 0\n",
+    "for ex in examples:\n",
+    "    msgs = ex.get('messages', [])\n",
+    "    user_msg = next((m['content'] for m in msgs if m['role']=='user'), '')\n",
+    "    chosen = next((m['content'] for m in msgs if m['role']=='assistant'), '')\n",
+    "    if not user_msg or not chosen or len(chosen) < 200:\n",
+    "        skipped += 1; continue\n",
+    "    try:\n",
+    "        rejected = generate_rejected(chosen)\n",
+    "        if not rejected or rejected == chosen or len(rejected) < 100:\n",
+    "            skipped += 1; continue\n",
+    "        pairs.append({'prompt': user_msg, 'chosen': chosen, 'rejected': rejected})\n",
+    "    except Exception as e:\n",
+    "        skipped += 1\n",
+    "\n",
+    "with open(PAIRS_OUT, 'w') as f:\n",
+    "    for p in pairs: f.write(json.dumps(p) + '\\n')\n",
+    "\n",
+    "print(f'Generated {len(pairs)} pairs, skipped {skipped}')\n",
+    "if pairs:\n",
+    "    s = pairs[0]\n",
+    "    print(f'\\nSample chosen: {len(s[\"chosen\"])} chars, rejected: {len(s[\"rejected\"])} chars')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Cell 6 — Load preference dataset\n",
+    "from datasets import load_dataset\n",
+    "ds = load_dataset('json', data_files=PAIRS_OUT)\n",
+    "print(f'Dataset: {len(ds[\"train\"])} pairs')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Cell 7 — Train SimPO (paper-optimal params)\n",
+    "from trl import CPOTrainer, CPOConfig\n",
+    "\n",
+    "config = CPOConfig(\n",
+    "    output_dir='/content/cipher-simpo-out',\n",
+    "    loss_type='simpo',\n",
+    "    cpo_alpha=0.0,        # pure SimPO\n",
+    "    simpo_gamma=2.5,      # paper-optimal\n",
+    "    beta=10.0,            # paper-optimal\n",
+    "    per_device_train_batch_size=1,\n",
+    "    gradient_accumulation_steps=128,  # effective batch 128 — #1 quality lever\n",
+    "    num_train_epochs=1,\n",
+    "    learning_rate=5e-6,\n",
+    "    bf16=True,\n",
+    "    max_length=4096,\n",
+    "    max_prompt_length=512,\n",
+    "    logging_steps=5,\n",
+    "    report_to='none',\n",
+    "    save_strategy='no',\n",
+    ")\n",
+    "\n",
+    "trainer = CPOTrainer(\n",
+    "    model=model,\n",
+    "    tokenizer=tokenizer,\n",
+    "    train_dataset=ds['train'],\n",
+    "    args=config,\n",
+    ")\n",
+    "\n",
+    "print(f'Pre-train VRAM: {torch.cuda.max_memory_allocated()/1e9:.1f}GB')\n",
+    "trainer.train()\n",
+    "print('SimPO training complete!')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Cell 8 — Merge + save + push to HF\n",
+    "MERGED_DIR = '/content/cipher-simpo-merged'\n",
+    "model.save_pretrained_merged(MERGED_DIR, tokenizer, save_method='merged_16bit')\n",
+    "print(f'Saved to {MERGED_DIR}')\n",
+    "\n",
+    "# Push to HF\n",
+    "from huggingface_hub import HfApi\n",
+    "api = HfApi()\n",
+    "api.create_repo('Auroraventures/cipher-simpo-merged', exist_ok=True)\n",
+    "api.upload_folder(\n",
+    "    folder_path=MERGED_DIR,\n",
+    "    repo_id='Auroraventures/cipher-simpo-merged',\n",
+    "    commit_message='Cipher SimPO merged - Stage 2 anti-slop',\n",
+    ")\n",
+    "print('Pushed to https://huggingface.co/Auroraventures/cipher-simpo-merged')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Cell 9 — Quick comparison test\n",
+    "FastLanguageModel.for_inference(model)\n",
+    "\n",
+    "test = 'Build a hero section with a Three.js particle system that responds to mouse movement.'\n",
+    "msgs = [{'role':'user','content':test}]\n",
+    "inputs = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_tensors='pt').to('cuda')\n",
+    "out = model.generate(input_ids=inputs, max_new_tokens=2048, temperature=0.7)\n",
+    "response = tokenizer.decode(out[0], skip_special_tokens=True)\n",
+    "\n",
+    "score = slop_score(response)\n",
+    "print(f'Slop score: {score[\"score\"]:.1f} (is_slop: {score[\"is_slop\"]})')\n",
+    "print(f'Signals: {score[\"signals\"]}')\n",
+    "print(f'Response length: {len(response)} chars')\n",
+    "print(f'Tailwind: {\"YES (BAD)\" if \"tailwindcss\" in response else \"no\"}')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "accelerator": "GPU",
+  "colab": {
+   "gpuType": "A100"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}