{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Cipher Stage 2 — SimPO Anti-Slop Training (Self-Contained)\n", "\n", "Pulls SFT-merged model from HF, generates preference pairs, trains SimPO, pushes result.\n", "\n", "**Runtime needed:** A100 (Colab Pro+). ~45-90 min total.\n", "**Output:** `Auroraventures/cipher-simpo-merged` on HuggingFace.\n", "\n", "Best-in-class params (paper-optimal): beta=10, gamma=2.5, grad_accum=128" ] }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": "# Cell 1 — Install\n!pip install -U unsloth trl datasets huggingface_hub -q\n!pip install --no-deps git+https://github.com/unslothai/unsloth.git@nightly -q\nimport os\nfrom google.colab import userdata\n\n# Try multiple secret names (case-insensitive fallback)\nhf_token = None\nfor name in ['HF_TOKEN', 'hf_token', 'HF_TOKE2', 'HF_TOKE', 'huggingface_token']:\n try:\n v = userdata.get(name)\n if v:\n hf_token = v\n print(f'Using token from secret: {name}')\n break\n except Exception:\n continue\n\nif not hf_token:\n raise RuntimeError(\n 'No HF token found. Add a Colab Secret named HF_TOKEN '\n '(left sidebar 🔑 icon) with toggle ON for this notebook.'\n )\n\nos.environ['HF_TOKEN'] = hf_token\nos.environ['WANDB_DISABLED'] = 'true'\nfrom huggingface_hub import login\nlogin(token=os.environ['HF_TOKEN'])\nprint('Setup done')" }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": [ "# Cell 2 — Load SFT-merged model from HF (~5 min, 62GB)\n", "import torch\n", "from unsloth import FastLanguageModel\n", "\n", "MODEL_ID = 'Auroraventures/cipher-sft-merged'\n", "MAX_SEQ = 8192\n", "\n", "model, tokenizer = FastLanguageModel.from_pretrained(\n", " model_name=MODEL_ID,\n", " max_seq_length=MAX_SEQ,\n", " load_in_4bit=True,\n", " dtype=None,\n", ")\n", "print(f'Loaded: VRAM {torch.cuda.max_memory_allocated()/1e9:.1f}GB')" ] }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": [ "# Cell 3 — Apply fresh LoRA for SimPO\n", "model = FastLanguageModel.get_peft_model(\n", " model,\n", " r=64, # paper-optimal for 31B with rsLoRA\n", " lora_alpha=64,\n", " target_modules=['q_proj','k_proj','v_proj','o_proj','gate_proj','up_proj','down_proj'],\n", " use_gradient_checkpointing='unsloth',\n", " use_rslora=True, # critical for r=64 stability\n", " random_state=42,\n", ")\n", "model.print_trainable_parameters()" ] }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": [ "# Cell 4 — Pull training data + slop tools from kr8tiv-training\n", "import subprocess, sys\n", "subprocess.run('rm -rf /content/kr8tiv-training && git clone --depth 1 https://github.com/kr8tiv-ai/kr8tiv-training.git /content/kr8tiv-training', shell=True, check=True)\n", "sys.path.insert(0, '/content/kr8tiv-training')\n", "from scripts.slop_detector import slop_score\n", "from scripts.rejected_generator import generate_rejected\n", "print('Tools loaded')" ] }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": "# Cell 5 — Generate SimPO preference pairs from existing SFT data\nimport json, traceback\nfrom pathlib import Path\n\nSFT_DATA = '/content/kr8tiv-training/data/prompts/awwwards-sft.jsonl'\nPAIRS_OUT = '/content/simpo_pairs.jsonl'\n\nexamples = []\nwith open(SFT_DATA) as f:\n for line in f:\n if line.strip():\n try: examples.append(json.loads(line))\n except: pass\n\nprint(f'Loaded {len(examples)} SFT examples')\n\n# Build preference pairs\npairs = []\nskipped = 0\nfirst_error = None\n\nfor ex in examples:\n msgs = ex.get('messages', [])\n user_msg = next((m['content'] for m in msgs if m['role']=='user'), '')\n chosen = next((m['content'] for m in msgs if m['role']=='assistant'), '')\n if not user_msg or not chosen or len(chosen) < 200:\n skipped += 1\n continue\n try:\n # Correct signature: (prompt, chosen_code)\n rejected = generate_rejected(user_msg, chosen)\n if not rejected or rejected == chosen or len(rejected) < 100:\n skipped += 1\n continue\n pairs.append({'prompt': user_msg, 'chosen': chosen, 'rejected': rejected})\n except Exception as e:\n if first_error is None:\n first_error = f'{type(e).__name__}: {e}'\n traceback.print_exc()\n skipped += 1\n\nwith open(PAIRS_OUT, 'w') as f:\n for p in pairs: f.write(json.dumps(p) + '\\n')\n\nprint(f'Generated {len(pairs)} pairs, skipped {skipped}')\nif first_error:\n print(f'First error encountered: {first_error}')\nif pairs:\n s = pairs[0]\n print(f'\\nSample chosen: {len(s[\"chosen\"])} chars, rejected: {len(s[\"rejected\"])} chars')\nelse:\n raise RuntimeError('Zero pairs generated — aborting.')" }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": [ "# Cell 6 — Load preference dataset\n", "from datasets import load_dataset\n", "ds = load_dataset('json', data_files=PAIRS_OUT)\n", "print(f'Dataset: {len(ds[\"train\"])} pairs')" ] }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": [ "# Cell 7 — Train SimPO (paper-optimal params)\n", "from trl import CPOTrainer, CPOConfig\n", "\n", "config = CPOConfig(\n", " output_dir='/content/cipher-simpo-out',\n", " loss_type='simpo',\n", " cpo_alpha=0.0, # pure SimPO\n", " simpo_gamma=2.5, # paper-optimal\n", " beta=10.0, # paper-optimal\n", " per_device_train_batch_size=1,\n", " gradient_accumulation_steps=128, # effective batch 128 — #1 quality lever\n", " num_train_epochs=1,\n", " learning_rate=5e-6,\n", " bf16=True,\n", " max_length=4096,\n", " max_prompt_length=512,\n", " logging_steps=5,\n", " report_to='none',\n", " save_strategy='no',\n", ")\n", "\n", "trainer = CPOTrainer(\n", " model=model,\n", " tokenizer=tokenizer,\n", " train_dataset=ds['train'],\n", " args=config,\n", ")\n", "\n", "print(f'Pre-train VRAM: {torch.cuda.max_memory_allocated()/1e9:.1f}GB')\n", "trainer.train()\n", "print('SimPO training complete!')" ] }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": [ "# Cell 8 — Merge + save + push to HF\n", "MERGED_DIR = '/content/cipher-simpo-merged'\n", "model.save_pretrained_merged(MERGED_DIR, tokenizer, save_method='merged_16bit')\n", "print(f'Saved to {MERGED_DIR}')\n", "\n", "# Push to HF\n", "from huggingface_hub import HfApi\n", "api = HfApi()\n", "api.create_repo('Auroraventures/cipher-simpo-merged', exist_ok=True)\n", "api.upload_folder(\n", " folder_path=MERGED_DIR,\n", " repo_id='Auroraventures/cipher-simpo-merged',\n", " commit_message='Cipher SimPO merged - Stage 2 anti-slop',\n", ")\n", "print('Pushed to https://huggingface.co/Auroraventures/cipher-simpo-merged')" ] }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], "source": [ "# Cell 9 — Quick comparison test\n", "FastLanguageModel.for_inference(model)\n", "\n", "test = 'Build a hero section with a Three.js particle system that responds to mouse movement.'\n", "msgs = [{'role':'user','content':test}]\n", "inputs = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_tensors='pt').to('cuda')\n", "out = model.generate(input_ids=inputs, max_new_tokens=2048, temperature=0.7)\n", "response = tokenizer.decode(out[0], skip_special_tokens=True)\n", "\n", "score = slop_score(response)\n", "print(f'Slop score: {score[\"score\"]:.1f} (is_slop: {score[\"is_slop\"]})')\n", "print(f'Signals: {score[\"signals\"]}')\n", "print(f'Response length: {len(response)} chars')\n", "print(f'Tailwind: {\"YES (BAD)\" if \"tailwindcss\" in response else \"no\"}')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "accelerator": "GPU", "colab": { "gpuType": "A100" } }, "nbformat": 4, "nbformat_minor": 4 }