Created using Colab

notebooks/01_model_training_and_fine_tuning.ipynb

@@ -5,7 +5,7 @@
     "colab": {
       "provenance": [],
       "gpuType": "T4",
-      "authorship_tag": "ABX9TyM5Fah77XuALjsYEAXaCFqE",
+      "authorship_tag": "ABX9TyMscbUSeB2qw1Lue5xsmWqV",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -44,13 +44,25 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 2,
+      "source": [
+        "%%capture install_output\n",
+        "!pip install optuna"
+      ],
+      "metadata": {
+        "id": "OwBert5NpQwF"
+      },
+      "execution_count": 5,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 6,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
         "id": "IGGkAlCRNlro",
-        "outputId": "919ad755-854c-49b1-a812-43d413128bb3"
+        "outputId": "778acab3-e0b5-4c40-a995-ded91103b0a4"
       },
       "outputs": [
         {
@@ -69,6 +81,9 @@
         "from torchvision import datasets, transforms, models\n",
         "from torch.utils.data import DataLoader\n",
         "import os\n",
+        "import optuna\n",
+        "import time\n",
+        "import copy\n",
         "import numpy as np\n",
         "\n",
         "# Configure the device\n",
@@ -122,9 +137,9 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "ikceEotXOU_1",
-        "outputId": "367466cb-bd4f-4de8-b8c6-1142abda4b49"
+        "outputId": "077395cc-6cf0-42f5-e0b3-65a1e322b5a8"
       },
-      "execution_count": 3,
+      "execution_count": 2,
       "outputs": [
         {
           "output_type": "stream",
@@ -135,11 +150,145 @@
         }
       ]
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## **Initializing Datasets and DataLoaders**\n",
+        "\n",
+        "With our transforms defined, we will use PyTorch's `ImageFolder` to automatically read our directory structure and assign the correct labels (0 for Normal, 1 for Pneumonia).\n",
+        "\n",
+        "We will then wrap these datasets in `DataLoaders`. DataLoaders handle the heavy lifting of batching the images, shuffling them, and pushing them to the GPU. We are setting the `batch_size` to 64 to fully utilize the Google Colab GPU memory."
+      ],
+      "metadata": {
+        "id": "odyLrQARmmn-"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# 1. Create the Dataset objects\n",
+        "train_dataset = datasets.ImageFolder(train_dir, transform=train_transforms)\n",
+        "val_dataset = datasets.ImageFolder(val_dir, transform=eval_transforms)\n",
+        "test_dataset = datasets.ImageFolder(test_dir, transform=eval_transforms)\n",
+        "\n",
+        "print(f\"Training images: {len(train_dataset)}\")\n",
+        "print(f\"Validation images: {len(val_dataset)}\")\n",
+        "print(f\"Testing images: {len(test_dataset)}\")\n",
+        "print(f\"Detected Classes: {train_dataset.classes}\")\n",
+        "\n",
+        "# 2. Create the DataLoaders\n",
+        "BATCH_SIZE = 64\n",
+        "\n",
+        "# We shuffle the training data so the model doesn't learn sequence patterns\n",
+        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2, pin_memory=True)\n",
+        "\n",
+        "# We do not need to shuffle validation/testing data\n",
+        "val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2, pin_memory=True)\n",
+        "test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2, pin_memory=True)\n",
+        "\n",
+        "print(\"DataLoaders are ready!\")"
+      ],
+      "metadata": {
+        "id": "I6obJOa1QYMy",
+        "outputId": "3a3b35dc-01e5-4980-bdfc-0958467a5f24",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 3,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training images: 5216\n",
+            "Validation images: 16\n",
+            "Testing images: 624\n",
+            "Detected Classes: ['NORMAL', 'PNEUMONIA']\n",
+            "DataLoaders are ready!\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## **Model Architecture and Class Weighting**\n",
+        "\n",
+        "Instead of training a model from scratch, we use **Transfer Learning**. We will download DenseNet121 (which already knows how to detect edges, textures, and shapes from millions of standard images) and replace its final classification layer with a custom one designed specifically for our two classes (Normal vs. Pneumonia).\n",
+        "\n",
+        "Simultaneously, we will calculate the inverse frequency of our classes to generate **Class Weights**, injecting them directly into the PyTorch loss function to penalize the model more heavily if it misclassifies a minority \"Normal\" scan."
+      ],
+      "metadata": {
+        "id": "MPPAHhh2nlVz"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# --- 1. Handling the Imbalance (Class Weights) ---\n",
+        "# ImageFolder automatically stores the label for every image in the .targets attribute\n",
+        "labels = train_dataset.targets\n",
+        "class_counts = np.bincount(labels)\n",
+        "total_samples = len(labels)\n",
+        "num_classes = len(train_dataset.classes)\n",
+        "\n",
+        "# Inverse frequency formula: Weight = Total / (NumClasses * ClassCount)\n",
+        "weights = total_samples / (num_classes * class_counts)\n",
+        "class_weights = torch.FloatTensor(weights).to(device)\n",
+        "\n",
+        "print(f\"Class Counts: NORMAL={class_counts[0]}, PNEUMONIA={class_counts[1]}\")\n",
+        "print(f\"Applied Weights: NORMAL={weights[0]:.4f}, PNEUMONIA={weights[1]:.4f}\\n\")\n",
+        "\n",
+        "# --- 2. Model Architecture (DenseNet121) ---\n",
+        "# Download the pre-trained neural network\n",
+        "model = models.densenet121(weights=models.DenseNet121_Weights.IMAGENET1K_V1)\n",
+        "\n",
+        "# Freeze the core convolutional layers so we don't destroy the pre-trained visual features\n",
+        "for param in model.parameters():\n",
+        "    param.requires_grad = False\n",
+        "\n",
+        "# Replace the final classification layer (originally 1000 classes) with our custom 2-class head\n",
+        "num_ftrs = model.classifier.in_features\n",
+        "\n",
+        "# Push the entire model architecture to the GPU\n",
+        "model = model.to(device)\n",
+        "\n",
+        "# --- 3. Loss Function and Optimizer ---\n",
+        "# Inject our calculated class weights into the loss function\n",
+        "criterion = nn.CrossEntropyLoss(weight=class_weights)\n",
+        "\n",
+        "# We only want the optimizer to update our newly added classifier layers\n",
+        "optimizer = optim.Adam(model.classifier.parameters(), lr=0.001)\n",
+        "\n",
+        "print(\"Model architecture, optimizer, and weighted loss function configured successfully!\")"
+      ],
+      "metadata": {
+        "id": "3cTqFqcvnVG2",
+        "outputId": "19533754-f990-4485-b848-d9c2be3f782b",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 7,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Class Counts: NORMAL=1341, PNEUMONIA=3875\n",
+            "Applied Weights: NORMAL=1.9448, PNEUMONIA=0.6730\n",
+            "\n",
+            "Model architecture, optimizer, and weighted loss function configured successfully!\n"
+          ]
+        }
+      ]
+    },
     {
       "cell_type": "code",
       "source": [],
       "metadata": {
-        "id": "I6obJOa1QYMy"
+        "id": "HrV6nUsjodCM"
       },
       "execution_count": null,
       "outputs": []