{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN baseline — ASL-HG, participant-disjoint\n",
    "\n",
    "Baseline tái lập: archive processed của ASL-HG, hash metadata đã publish, khử exact duplicate và split theo người tham gia (8/1/1). Không dùng split train/test dựng sẵn của tác giả vì protocol này khóa validation và test theo participant.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip -q install 'huggingface-hub>=0.25' pandas pyarrow scikit-learn matplotlib seaborn\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from datetime import datetime, timezone\n",
    "import hashlib, json, os, random, re, shutil, zipfile\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "from huggingface_hub import snapshot_download\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
    "\n",
    "EXPERIMENT_ID = 'cnn-001-participant-disjoint'\n",
    "DATASET_REPO = 'hnam25/asl-hand-gesture-images'\n",
    "DATASET_REVISION = '8f36ac00ece6dfce94410a980a839d93a912d366'\n",
    "AUDIT_DIRECTORY = 'metadata/colab-audit-2026-08-10'\n",
    "PROCESSED_ARCHIVE = 'ASL_HG_36000/ASL_Processed_Images.zip'\n",
    "SEED, IMAGE_SIZE, BATCH_SIZE, EPOCHS, LEARNING_RATE, DROPOUT = 42, 128, 64, 30, 1e-3, .3\n",
    "CLASSES = [str(i) for i in range(10)] + [chr(i) for i in range(ord('A'), ord('Z') + 1)]\n",
    "ROOT = Path('/content/asl-cnn-baseline')\n",
    "HF_ROOT, PROCESSED, OUTPUTS = ROOT/'hf', ROOT/'processed', ROOT/'outputs'\n",
    "for directory in (HF_ROOT, PROCESSED, OUTPUTS/'models', OUTPUTS/'metrics', OUTPUTS/'figures', OUTPUTS/'logs', OUTPUTS/'metadata'):\n",
    "    directory.mkdir(parents=True, exist_ok=True)\n",
    "random.seed(SEED); np.random.seed(SEED); tf.keras.utils.set_random_seed(SEED)\n",
    "print({'tensorflow': tf.__version__, 'gpus': [d.name for d in tf.config.list_physical_devices('GPU')], 'experiment_id': EXPERIMENT_ID})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sha256_file(path, chunk_size=8 * 1024 * 1024):\n",
    "    digest = hashlib.sha256()\n",
    "    with Path(path).open('rb') as handle:\n",
    "        for chunk in iter(lambda: handle.read(chunk_size), b''):\n",
    "            digest.update(chunk)\n",
    "    return digest.hexdigest()\n",
    "\n",
    "snapshot_download(repo_id=DATASET_REPO, repo_type='dataset', revision=DATASET_REVISION, local_dir=HF_ROOT, allow_patterns=[PROCESSED_ARCHIVE, f'{AUDIT_DIRECTORY}/**'])\n",
    "archive = HF_ROOT / PROCESSED_ARCHIVE\n",
    "audit_root = HF_ROOT / AUDIT_DIRECTORY\n",
    "audit_manifest = json.loads((audit_root/'cache_manifest.json').read_text())\n",
    "if not archive.is_file(): raise RuntimeError(f'Missing {PROCESSED_ARCHIVE}')\n",
    "with zipfile.ZipFile(archive) as z:\n",
    "    for member in z.infolist():\n",
    "        target = (PROCESSED/member.filename).resolve()\n",
    "        if PROCESSED.resolve() not in target.parents and target != PROCESSED.resolve(): raise RuntimeError(f'Unsafe ZIP member: {member.filename}')\n",
    "    z.extractall(PROCESSED)\n",
    "processed_archive_sha256 = sha256_file(archive)\n",
    "audit = pd.read_csv(audit_root/'audit.csv')\n",
    "print({'processed_archive_sha256': processed_archive_sha256, 'audit_totals': audit_manifest['totals']})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Match every processed image to audited raw-image hash, then retain one representative per exact hash.\n",
    "image_paths = sorted(path for path in PROCESSED.rglob('*') if path.suffix.lower() in {'.jpg', '.jpeg', '.png', '.bmp', '.webp'})\n",
    "records = []\n",
    "subject_pattern = re.compile(r'^P(\\d+)_')\n",
    "for path in image_paths:\n",
    "    parts = path.parts\n",
    "    label = next((part for part in reversed(parts[:-1]) if part in CLASSES), None)\n",
    "    if label is None: raise RuntimeError(f'Cannot infer label from {path}')\n",
    "    match = subject_pattern.match(path.name)\n",
    "    if not match: raise RuntimeError(f'Cannot infer participant from {path.name}')\n",
    "    records.append({'label': label, 'processed_path': str(path), 'relative_path': f'{label}/{path.name}', 'participant': f'P{match.group(1)}'})\n",
    "processed = pd.DataFrame(records)\n",
    "usable = processed.merge(audit[audit.status == 'ok'][['relative_path', 'label', 'sha256']], on=['relative_path', 'label'], how='left', validate='one_to_one')\n",
    "if len(usable) != 36000 or usable.sha256.isna().any(): raise RuntimeError('Processed archive does not match the pinned audit metadata.')\n",
    "if (usable.groupby('sha256').label.nunique() > 1).any(): raise RuntimeError('Conflicting labels for an exact raw-image hash.')\n",
    "usable = usable.sort_values(['sha256', 'relative_path'], kind='stable').reset_index(drop=True)\n",
    "usable['duplicate_group_size'] = usable.groupby('sha256').sha256.transform('size')\n",
    "usable['canonical_relative_path'] = usable.groupby('sha256').relative_path.transform('first')\n",
    "usable['is_canonical'] = usable.relative_path.eq(usable.canonical_relative_path)\n",
    "usable.to_csv(OUTPUTS/'metadata'/'deduplication_manifest.csv', index=False)\n",
    "before_dedup = len(usable); usable = usable[usable.is_canonical].copy()\n",
    "participants = sorted(usable.participant.unique(), key=lambda value: int(value[1:]))\n",
    "if len(participants) != 10 or set(usable.participant) != set(participants): raise RuntimeError(f'Expected P1-P10, got {participants}')\n",
    "rng = np.random.default_rng(SEED); ordered = list(rng.permutation(participants))\n",
    "train_participants, validation_participant, test_participant = sorted(ordered[:8]), ordered[8], ordered[9]\n",
    "train = usable[usable.participant.isin(train_participants)].copy()\n",
    "validation = usable[usable.participant.eq(validation_participant)].copy()\n",
    "test = usable[usable.participant.eq(test_participant)].copy()\n",
    "for name, frame in {'train': train, 'validation': validation, 'test': test}.items(): frame[['processed_path', 'label', 'participant', 'sha256']].to_csv(OUTPUTS/'metadata'/f'{name}.csv', index=False)\n",
    "if set(train.sha256) & set(validation.sha256) or set(train.sha256) & set(test.sha256) or set(validation.sha256) & set(test.sha256): raise RuntimeError('Hash leakage detected.')\n",
    "if set(train.participant) & set(validation.participant) or set(train.participant) & set(test.participant) or set(validation.participant) & set(test.participant): raise RuntimeError('Participant leakage detected.')\n",
    "split_manifest = {'experiment_id': EXPERIMENT_ID, 'policy': 'participant-disjoint 8/1/1; one canonical representative per exact raw-image SHA-256', 'seed': SEED, 'participants': {'train': train_participants, 'validation': validation_participant, 'test': test_participant}, 'counts': {'before_deduplication': before_dedup, 'after_deduplication': len(usable), 'removed_exact_duplicates': before_dedup-len(usable), 'train': len(train), 'validation': len(validation), 'test': len(test)}}\n",
    "(OUTPUTS/'metadata'/'split_manifest.json').write_text(json.dumps(split_manifest, indent=2), encoding='utf-8')\n",
    "print(json.dumps(split_manifest, indent=2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_index = {label: index for index, label in enumerate(CLASSES)}\n",
    "def make_dataset(frame, training=False):\n",
    "    ds = tf.data.Dataset.from_tensor_slices((frame.processed_path.values, frame.label.map(label_index).values))\n",
    "    if training: ds = ds.shuffle(len(frame), seed=SEED, reshuffle_each_iteration=True)\n",
    "    def load(path, label):\n",
    "        image = tf.io.decode_image(tf.io.read_file(path), channels=3, expand_animations=False); image.set_shape([None, None, 3])\n",
    "        image = tf.image.resize(tf.cast(image, tf.float32), (IMAGE_SIZE, IMAGE_SIZE))\n",
    "        return image, tf.one_hot(label, len(CLASSES))\n",
    "    return ds.map(load, num_parallel_calls=tf.data.AUTOTUNE).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "train_ds, validation_ds, test_ds = make_dataset(train, True), make_dataset(validation), make_dataset(test)\n",
    "def build_cnn():\n",
    "    inputs = tf.keras.Input((IMAGE_SIZE, IMAGE_SIZE, 3), name='image')\n",
    "    x = tf.keras.layers.Rescaling(1/255, name='rescale')(inputs)\n",
    "    for filters in (32, 64, 128):\n",
    "        x = tf.keras.layers.Conv2D(filters, 3, padding='same', use_bias=False)(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x); x = tf.keras.layers.ReLU()(x)\n",
    "        x = tf.keras.layers.Conv2D(filters, 3, padding='same', use_bias=False)(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x); x = tf.keras.layers.ReLU()(x)\n",
    "        x = tf.keras.layers.MaxPooling2D()(x); x = tf.keras.layers.Dropout(DROPOUT / 2)(x)\n",
    "    x = tf.keras.layers.GlobalAveragePooling2D(name='global_average_pooling')(x)\n",
    "    x = tf.keras.layers.Dense(256, activation='relu', name='features')(x)\n",
    "    x = tf.keras.layers.Dropout(DROPOUT, name='dropout')(x)\n",
    "    outputs = tf.keras.layers.Dense(len(CLASSES), activation='softmax', name='classification')(x)\n",
    "    return tf.keras.Model(inputs, outputs, name='cnn_asl')\n",
    "\n",
    "model = build_cnn(); model.compile(optimizer=tf.keras.optimizers.Adam(LEARNING_RATE), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "with (OUTPUTS/'models'/'model_summary.txt').open('w') as handle: model.summary(print_fn=lambda line: handle.write(line+'\\n'))\n",
    "checkpoint = OUTPUTS/'models'/'cnn_001_participant_disjoint.keras'\n",
    "callbacks = [tf.keras.callbacks.ModelCheckpoint(checkpoint, monitor='val_accuracy', save_best_only=True), tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True), tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=2, factor=.2), tf.keras.callbacks.CSVLogger(OUTPUTS/'logs'/'training_history.csv')]\n",
    "history = model.fit(train_ds, validation_data=validation_ds, epochs=EPOCHS, verbose=2, callbacks=callbacks)\n",
    "pd.DataFrame(history.history).to_csv(OUTPUTS/'logs'/'training_history.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best = tf.keras.models.load_model(checkpoint)\n",
    "probability = best.predict(test_ds, verbose=1); predicted = probability.argmax(1); truth = test.label.map(label_index).to_numpy()\n",
    "report = classification_report(truth, predicted, labels=range(36), target_names=CLASSES, output_dict=True, zero_division=0)\n",
    "matrix = confusion_matrix(truth, predicted, labels=range(36))\n",
    "pd.DataFrame(report).T.to_csv(OUTPUTS/'metrics'/'classification_report.csv')\n",
    "pd.DataFrame(matrix, index=CLASSES, columns=CLASSES).to_csv(OUTPUTS/'metrics'/'confusion_matrix.csv')\n",
    "o, zero = label_index['O'], label_index['0']\n",
    "summary = {'experiment_id': EXPERIMENT_ID, 'test_accuracy': float(accuracy_score(truth, predicted)), 'macro_precision': report['macro avg']['precision'], 'macro_recall': report['macro avg']['recall'], 'macro_f1': report['macro avg']['f1-score'], 'O_recall': report['O']['recall'], '0_recall': report['0']['recall'], 'O_to_0': int(matrix[o, zero]), '0_to_O': int(matrix[zero, o]), 'best_validation_accuracy': float(max(history.history['val_accuracy'])), 'epochs_ran': len(history.history['loss'])}\n",
    "(OUTPUTS/'metrics'/'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8')\n",
    "plt.figure(figsize=(16, 13)); sns.heatmap(matrix, cmap='Blues', xticklabels=CLASSES, yticklabels=CLASSES); plt.xlabel('Predicted'); plt.ylabel('True'); plt.tight_layout(); plt.savefig(OUTPUTS/'figures'/'confusion_matrix.png', dpi=180); plt.close()\n",
    "config = {'experiment_id': EXPERIMENT_ID, 'created_at_utc': datetime.now(timezone.utc).isoformat(), 'dataset_repo': DATASET_REPO, 'dataset_revision': DATASET_REVISION, 'processed_archive': PROCESSED_ARCHIVE, 'processed_archive_sha256': processed_archive_sha256, 'audit_manifest': audit_manifest, 'split_manifest': split_manifest, 'model': {'architecture': 'CNN from scratch: 3 x [Conv-BN-ReLU-Conv-BN-ReLU-MaxPool]', 'image_size': IMAGE_SIZE, 'batch_size': BATCH_SIZE, 'epochs_max': EPOCHS, 'learning_rate': LEARNING_RATE, 'dropout': DROPOUT}, 'tensorflow': tf.__version__}\n",
    "(OUTPUTS/'metadata'/'experiment_config.json').write_text(json.dumps(config, indent=2), encoding='utf-8')\n",
    "os.system(f\"pip freeze > {OUTPUTS/'metadata'/'environment.txt'}\")\n",
    "print(json.dumps(summary, indent=2))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Publish\n",
    "\n",
    "Download `outputs/` về local. Tạo một Hugging Face model repo và upload nguyên thư mục output cùng notebook này; model card phải nêu rõ participant-disjoint protocol, revision dữ liệu, SHA archive và số liệu test.\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.x"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}