Add dataset options and replacement loader cell

DATASET_OPTIONS.md

@@ -0,0 +1,163 @@
+# Dataset Options for the Fine-Tuning Notebooks
+
+Use these values in the config cell:
+
+```python
+DATASET_CHOICE = "cybersecurity"
+# DATASET_CHOICE = "code_corpus"
+# DATASET_CHOICE = "ultrachat"
+# DATASET_CHOICE = "openhermes"
+# DATASET_CHOICE = "sharegpt_en"
+# DATASET_CHOICE = "sharegpt_de"
+# DATASET_CHOICE = "sharegpt_hi"
+# DATASET_CHOICE = "custom_mix"
+```
+
+## Recommended choices
+
+| Choice | Dataset | Best for |
+|---|---|---|
+| `cybersecurity` | `AlicanKiraz0/Cybersecurity-Dataset-Fenrir-v2.1` + `Trendyol/Trendyol-Cybersecurity-Instruction-Tuning-Dataset` | ethical hacking / cyber defense |
+| `code_corpus` | `krystv/code-corpus-llm-training` | code explanation, completion, refactoring |
+| `ultrachat` | `HuggingFaceH4/ultrachat_200k` | general assistant chat |
+| `openhermes` | `teknium/OpenHermes-2.5` | reasoning, coding, general instruction |
+| `sharegpt_en` | `deepmage121/ShareGPT_multilingual`, English split | English multi-turn dialogue |
+| `sharegpt_de` | `deepmage121/ShareGPT_multilingual`, German translated split | German dialogue |
+| `sharegpt_hi` | `deepmage121/ShareGPT_multilingual`, Hindi translated split | Hindi dialogue |
+| `custom_mix` | mix any supported datasets | hybrid assistant |
+
+## Full replacement dataset cell
+
+If a notebook does not yet show all choices, replace its dataset-loading cell with this one:
+
+```python
+from datasets import load_dataset, concatenate_datasets
+
+SYSTEM_SAFE = "You are a cybersecurity education assistant. Provide defensive, ethical, and authorized security guidance only. Refuse harmful or unauthorized requests."
+
+# Uncomment one in config cell:
+# DATASET_CHOICE = "cybersecurity"
+# DATASET_CHOICE = "code_corpus"
+# DATASET_CHOICE = "ultrachat"
+# DATASET_CHOICE = "openhermes"
+# DATASET_CHOICE = "sharegpt_en"
+# DATASET_CHOICE = "sharegpt_de"
+# DATASET_CHOICE = "sharegpt_hi"
+# DATASET_CHOICE = "custom_mix"
+
+CUSTOM_DATASETS = [
+    # (dataset_id, split, n_rows, format_type)
+    # format_type: cyber_sua | ultrachat | conversations | code
+    ("AlicanKiraz0/Cybersecurity-Dataset-Fenrir-v2.1", "train", 5000, "cyber_sua"),
+    ("Trendyol/Trendyol-Cybersecurity-Instruction-Tuning-Dataset", "train", 5000, "cyber_sua"),
+    ("teknium/OpenHermes-2.5", "train", 10000, "conversations"),
+    ("krystv/code-corpus-llm-training", "train", 10000, "code"),
+]
+
+def convert_sua(example):
+    return {"messages": [
+        {"role": "system", "content": example.get("system") or SYSTEM_SAFE},
+        {"role": "user", "content": example["user"]},
+        {"role": "assistant", "content": example["assistant"]},
+    ]}
+
+def convert_ultrachat(example):
+    return {"messages": example["messages"]}
+
+def convert_conversations(example):
+    msgs = []
+    sys_prompt = example.get("system_prompt", "") or example.get("system", "")
+    if sys_prompt:
+        msgs.append({"role": "system", "content": sys_prompt})
+    for turn in example["conversations"]:
+        role = "user" if turn.get("from") in ("human", "user") else "assistant"
+        content = turn.get("value", "")
+        if content:
+            msgs.append({"role": role, "content": content})
+    return {"messages": msgs}
+
+def convert_code(example):
+    code_text = example.get("text", "")
+    lang = example.get("language", "")
+    repo = example.get("repo", "unknown")
+    prompt = f"Explain, review, and improve this {lang} code snippet from repo {repo}. Focus on correctness, security, performance, and clarity."
+    return {"messages": [
+        {"role": "system", "content": "You are a careful coding assistant focused on secure, correct, maintainable code."},
+        {"role": "user", "content": prompt},
+        {"role": "assistant", "content": code_text},
+    ]}
+
+def apply_converter(ds, fmt):
+    if fmt == "cyber_sua":
+        return ds.map(convert_sua, remove_columns=ds.column_names)
+    if fmt == "ultrachat":
+        return ds.map(convert_ultrachat, remove_columns=ds.column_names)
+    if fmt == "conversations":
+        return ds.map(convert_conversations, remove_columns=ds.column_names)
+    if fmt == "code":
+        return ds.map(convert_code, remove_columns=ds.column_names)
+    raise ValueError(f"Unknown format_type: {fmt}")
+
+all_datasets = []
+
+if DATASET_CHOICE == "cybersecurity":
+    ds1 = load_dataset("AlicanKiraz0/Cybersecurity-Dataset-Fenrir-v2.1", split="train")
+    ds2 = load_dataset("Trendyol/Trendyol-Cybersecurity-Instruction-Tuning-Dataset", split="train")
+    all_datasets = [apply_converter(ds1, "cyber_sua"), apply_converter(ds2, "cyber_sua")]
+
+elif DATASET_CHOICE == "code_corpus":
+    ds = load_dataset("krystv/code-corpus-llm-training", split="train")
+    all_datasets = [apply_converter(ds, "code")]
+
+elif DATASET_CHOICE == "ultrachat":
+    ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft")
+    all_datasets = [apply_converter(ds, "ultrachat")]
+
+elif DATASET_CHOICE == "openhermes":
+    ds = load_dataset("teknium/OpenHermes-2.5", split="train")
+    all_datasets = [apply_converter(ds, "conversations")]
+
+elif DATASET_CHOICE.startswith("sharegpt_"):
+    split_map = {
+        "sharegpt_en": "english",
+        "sharegpt_de": "german_4b_translated",
+        "sharegpt_hi": "hindi_27b_translated",
+    }
+    ds = load_dataset("deepmage121/ShareGPT_multilingual", split=split_map[DATASET_CHOICE])
+    all_datasets = [apply_converter(ds, "conversations")]
+
+elif DATASET_CHOICE == "custom_mix":
+    for ds_id, split, n_rows, fmt in CUSTOM_DATASETS:
+        print(f"Loading {ds_id} split={split} rows={n_rows} format={fmt}")
+        ds = load_dataset(ds_id, split=split)
+        if n_rows and len(ds) > n_rows:
+            ds = ds.shuffle(seed=SEED).select(range(n_rows))
+        all_datasets.append(apply_converter(ds, fmt))
+
+else:
+    raise ValueError(f"Unknown DATASET_CHOICE: {DATASET_CHOICE}")
+
+dataset = concatenate_datasets(all_datasets) if len(all_datasets) > 1 else all_datasets[0]
+print(f"Rows before sampling: {len(dataset):,}")
+
+if SAMPLE_SIZE and len(dataset) > SAMPLE_SIZE:
+    dataset = dataset.shuffle(seed=SEED).select(range(SAMPLE_SIZE))
+    print(f"Rows after sampling: {len(dataset):,}")
+
+print("Sample:")
+for m in dataset[0]["messages"]:
+    print(m["role"], ":", m["content"][:160].replace("\n", " "))
+```
+
+## Notes
+
+- For `code_corpus`, use lower `MAX_SEQ_LENGTH` at first because code files can be long:
+
+```python
+MAX_SEQ_LENGTH = 2048
+SAMPLE_SIZE = 10000
+MAX_STEPS = 500
+```
+
+- For `custom_mix`, keep each source smaller first to verify training starts.
+- For Kaggle T4, LFM2.5 is still the safest model for all dataset options.