studies_classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "studies-classifier.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "X9tu5eAf7mXF"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "NzqxHIh4OCdW"
      },
      "source": [
        "# Incremental learning on image classification\n",
        "**Ablation studies**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wBHSznCZxpNB"
      },
      "source": [
        "## Libraries and packages\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Kvngrp1R1nO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4eQ6O12jxMFf",
        "colab": {}
      },
      "source": [
        "!pip3 install 'torch==1.4.0'\n",
        "!pip3 install 'torchvision==0.5.0'\n",
        "!pip3 install 'Pillow-SIMD'\n",
        "!pip3 install 'tqdm'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xAYXtIdpx0Yy",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import urllib\n",
        "import logging\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.init as init\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, Subset, DataLoader, ConcatDataset\n",
        "from torch.backends import cudnn\n",
        "\n",
        "import torchvision\n",
        "from torchvision import transforms\n",
        "from torchvision.models import resnet34\n",
        "\n",
        "from PIL import Image\n",
        "from tqdm import tqdm\n",
        "\n",
        "from copy import deepcopy\n",
        "\n",
        "from sklearn.metrics import confusion_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "09iWc_oCotu2",
        "colab": {}
      },
      "source": [
        "# GitHub credentials for cloning private repository\n",
        "username = ''\n",
        "password = ''\n",
        "\n",
        "# Download packages from repository\n",
        "password = urllib.parse.quote(password)\n",
        "!git clone https://$username:$password@github.com/manuelemacchia/incremental-learning-image-classification.git\n",
        "password = ''\n",
        "\n",
        "!mv -v incremental-learning-image-classification/* .\n",
        "!rm -rf incremental-learning-image-classification README.md"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QPLViftqtC3I",
        "colab": {}
      },
      "source": [
        "from data.cifar100 import Cifar100\n",
        "from model.resnet_cifar import resnet32\n",
        "from model.manager import Manager\n",
        "from model.icarl import Exemplars\n",
        "from model.icarl import iCaRL\n",
        "from utils import plot"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8QYf4IU5iAI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "import time\n",
        "from google.colab import files\n",
        "\n",
        "if not os.path.isdir('./obj'):\n",
        "    !mkdir 'obj'\n",
        "\n",
        "def obj_save(obj, name):\n",
        "    with open('obj/'+ name + '.pkl', 'wb') as f:\n",
        "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
        "    \n",
        "    time.sleep(5)\n",
        "\n",
        "    files.download('obj/'+ name + '.pkl') \n",
        "\n",
        "def obj_load(name):\n",
        "    with open('obj/' + name + '.pkl', 'rb') as f:\n",
        "        return pickle.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "j12pgffMR6Qv"
      },
      "source": [
        "## Arguments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JwE0x8gkSisn",
        "colab": {}
      },
      "source": [
        "# Directories\n",
        "DATA_DIR = 'data'       # Directory where the dataset will be downloaded\n",
        "\n",
        "# Settings\n",
        "DEVICE = 'cuda'\n",
        "\n",
        "# Dataset\n",
        "\n",
        "RANDOM_STATE = None\n",
        "\n",
        "RANDOM_STATES = [658, 423, 422]      # For reproducibility of results                        \n",
        "                                     # Note: different random states give very different\n",
        "                                     # splits and therefore very different results.\n",
        "\n",
        "NUM_CLASSES = 100       # Total number of classes\n",
        "NUM_BATCHES = 10\n",
        "CLASS_BATCH_SIZE = 10   # Size of batch of classes for incremental learning\n",
        "\n",
        "VAL_SIZE = 0.1          # Proportion of validation set with respect to training set (between 0 and 1)\n",
        "\n",
        "# Training\n",
        "BATCH_SIZE = 64         # Batch size (iCaRL sets this to 128)\n",
        "LR = 2                  # Initial learning rate\n",
        "                       \n",
        "MOMENTUM = 0.9          # Momentum for stochastic gradient descent (SGD)\n",
        "WEIGHT_DECAY = 1e-5     # Weight decay from iCaRL\n",
        "\n",
        "NUM_RUNS = 3            # Number of runs of every method\n",
        "                        # Note: this should be at least 3 to have a fair benchmark\n",
        "\n",
        "NUM_EPOCHS = 70         # Total number of training epochs\n",
        "MILESTONES = [49, 63]   # Step down policy from iCaRL (MultiStepLR)\n",
        "                        # Decrease the learning rate by gamma at each milestone\n",
        "GAMMA = 0.2             # Gamma factor from iCaRL"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rF0ypxGognNR",
        "colab_type": "text"
      },
      "source": [
        "## Data preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4mf04UjEgmPG",
        "colab": {}
      },
      "source": [
        "# Transformations for Learning Without Forgetting\n",
        "train_transform = transforms.Compose([transforms.RandomCrop(32, padding=4),\n",
        "                                      transforms.RandomHorizontalFlip(),\n",
        "                                      transforms.ToTensor(), # Turn PIL Image to torch.Tensor\n",
        "                                      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
        "])\n",
        "\n",
        "test_transform = transforms.Compose([transforms.ToTensor(),\n",
        "                                     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))                                    \n",
        "])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Y9Oq44dxgmPN",
        "colab": {}
      },
      "source": [
        "train_subsets = [[] for i in range(NUM_RUNS)]\n",
        "val_subsets = [[] for i in range(NUM_RUNS)]\n",
        "test_subsets = [[] for i in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    for split_i in range(CLASS_BATCH_SIZE):\n",
        "        if run_i+split_i == 0: # Download dataset only at first instantiation\n",
        "            download = True\n",
        "        else:\n",
        "            download = False\n",
        "\n",
        "        # Create CIFAR100 dataset\n",
        "        train_dataset = Cifar100(DATA_DIR, train=True, download=download, random_state=RANDOM_STATES[run_i], transform=train_transform)\n",
        "        test_dataset = Cifar100(DATA_DIR, train=False, download=False, random_state=RANDOM_STATES[run_i], transform=test_transform)\n",
        "    \n",
        "        # Subspace of CIFAR100 of 10 classes\n",
        "        train_dataset.set_classes_batch(train_dataset.batch_splits[split_i]) \n",
        "        test_dataset.set_classes_batch([test_dataset.batch_splits[i] for i in range(0, split_i+1)])\n",
        "\n",
        "        # Define train and validation indices\n",
        "        train_indices, val_indices = train_dataset.train_val_split(VAL_SIZE, RANDOM_STATES[run_i])\n",
        "\n",
        "        # Define subsets\n",
        "        train_subsets[run_i].append(Subset(train_dataset, train_indices))\n",
        "        val_subsets[run_i].append(Subset(train_dataset, val_indices))\n",
        "        test_subsets[run_i].append(test_dataset)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x0oEuUQPo2Nb",
        "colab_type": "text"
      },
      "source": [
        "## Classifiers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X9tu5eAf7mXF",
        "colab_type": "text"
      },
      "source": [
        "### K-nearest neighbors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fnolqQQ72Gt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.model_selection  import ParameterGrid\n",
        "from copy import deepcopy\n",
        "\n",
        "class iCaRLwithKNN(iCaRL):\n",
        "    def classifier_fit(self, train_dataset, val_dataset, params, only_exemplars=False):\n",
        "        \"\"\"Fit classifier on the union of training dataset and exemplars,\n",
        "        and validate it on val_dataset.\"\"\"\n",
        "\n",
        "        if only_exemplars:\n",
        "            fit_dataset = Exemplars(self.exemplars, self.train_transform)\n",
        "        else:\n",
        "            # Union of training dataset and exemplars\n",
        "            exemplars_dataset = Exemplars(self.exemplars, self.train_transform)\n",
        "            fit_dataset = ConcatDataset([exemplars_dataset, train_dataset])\n",
        "\n",
        "        # Convert dataset to numpy format\n",
        "        # X contains training samples, y contains labels\n",
        "        X, y = self.dataset_to_numpy(fit_dataset)\n",
        "\n",
        "        # Extract features from the training dataset\n",
        "        X_features = self.extract_features(torch.tensor(X, dtype=torch.float))\n",
        "        for i in range(X_features.size(0)):\n",
        "            X_features[i] = X_features[i]/X_features[i].norm()\n",
        "        X_features = X_features.to('cpu').numpy()\n",
        "\n",
        "        # Scale training features to range [0, 1] individually\n",
        "        self.scaler = MinMaxScaler()\n",
        "        self.scaler.fit(X_features)\n",
        "        X_features = self.scaler.transform(X_features)\n",
        "\n",
        "        # Initialize classifier\n",
        "        self.clf = KNeighborsClassifier()\n",
        "\n",
        "        # Run validation\n",
        "        best_clf = None\n",
        "        best_grid = None\n",
        "        best_score = 0\n",
        "\n",
        "        X_test, y_test = self.dataset_to_numpy(val_dataset)\n",
        "        X_test_features = self.extract_features(torch.tensor(X_test, dtype=torch.float))\n",
        "        for i in range(X_test_features.size(0)):\n",
        "            X_test_features[i] = X_test_features[i]/X_test_features[i].norm()\n",
        "        X_test_features = X_test_features.to('cpu').numpy()\n",
        "\n",
        "        X_test_features = self.scaler.transform(X_test_features)\n",
        "\n",
        "        for grid in ParameterGrid(params):\n",
        "            self.clf.set_params(**grid)\n",
        "            self.clf.fit(X_features, y)\n",
        "            y_pred = self.clf.predict(X_test_features)\n",
        "            score = accuracy_score(y_test, y_pred)\n",
        "\n",
        "            if score > best_score:\n",
        "                best_clf = deepcopy(self.clf)\n",
        "                best_score = score\n",
        "                best_grid = grid\n",
        "\n",
        "        # Set the classifier to the best clf found in validation\n",
        "        self.clf = best_clf\n",
        "\n",
        "        print(f\"Best classifier: {best_grid} with score {best_score}\")\n",
        "\n",
        "        return best_grid\n",
        "\n",
        "    def classifier_predict(self, test_dataset):\n",
        "        \"\"\"Predict labels of test_dataset.\"\"\"\n",
        "\n",
        "        X_test, y_test = self.dataset_to_numpy(test_dataset)\n",
        "\n",
        "        # Extract features from the test set\n",
        "        X_test_features = self.extract_features(torch.tensor(X_test, dtype=torch.float))\n",
        "        for i in range(X_test_features.size(0)):\n",
        "            X_test_features[i] = X_test_features[i]/X_test_features[i].norm()\n",
        "        X_test_features = X_test_features.to('cpu').numpy()\n",
        "\n",
        "        X_test_features = self.scaler.transform(X_test_features)\n",
        "        \n",
        "        y_pred = self.clf.predict(X_test_features)\n",
        "\n",
        "        return y_test, y_pred\n",
        "\n",
        "    def dataset_to_numpy(self, dataset):\n",
        "        # Preallocate arrays\n",
        "        X = np.zeros((len(dataset), 3, 32, 32))\n",
        "        y = np.zeros(len(dataset), dtype=int)\n",
        "\n",
        "        dataloader = DataLoader(dataset, batch_size=1)\n",
        "\n",
        "        for idx, (image, labels) in enumerate(dataloader):\n",
        "            X[idx] = image[0].numpy()\n",
        "            y[idx] = labels.numpy()[0]\n",
        "\n",
        "        return X, y\n",
        "\n",
        "    def test_knn(self, test_dataset, train_dataset, val_dataset, params, only_exemplars=False):\n",
        "        \"\"\"Test the model.\n",
        "\n",
        "        Args:\n",
        "            test_dataset: dataset on which to test the network\n",
        "            train_dataset: training set used to train the last split\n",
        "            params: parameter grid on which to perform hyperparameter tuning\n",
        "        Returns:\n",
        "            accuracy (float): accuracy of the model on the test set\n",
        "        \"\"\"\n",
        "\n",
        "        self.net.train(False)\n",
        "        if self.best_net is not None: self.best_net.train(False)  # Set Network to evaluation mode\n",
        "        if self.old_net is not None: self.old_net.train(False)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            # Use val_dataset as validation set for hyperparameter tuning.\n",
        "            best_grid = self.classifier_fit(train_dataset, val_dataset, params, only_exemplars)\n",
        "            y_truth, y_pred = self.classifier_predict(test_dataset)\n",
        "            accuracy = accuracy_score(y_truth, y_pred)\n",
        "\n",
        "            if only_exemplars:\n",
        "                print(f\"Test accuracy (iCaRL with KNN only exemplars): {accuracy} \")\n",
        "            else:\n",
        "                print(f\"Test accuracy (iCaRL with KNN all available data): {accuracy} \")\n",
        "\n",
        "        return accuracy, best_grid, torch.tensor(y_truth), torch.tensor(y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uNc4tS3-kO_7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "NUM_EPOCHS = 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XwLskgOPHZoD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "logs = [[] for _ in range(NUM_RUNS)]\n",
        "\n",
        "params = {\n",
        "    'n_neighbors': [3, 5, 7, 9],\n",
        "    'weights': ['uniform', 'distance']\n",
        "}\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    net = resnet32()\n",
        "    icarl_knn = iCaRLwithKNN(DEVICE, net, LR, MOMENTUM, WEIGHT_DECAY, MILESTONES, GAMMA, NUM_EPOCHS, BATCH_SIZE, train_transform, test_transform)\n",
        "\n",
        "    for split_i in range(10):\n",
        "        print(f\"## Split {split_i} of run {run_i} ##\")\n",
        "        \n",
        "        icarl_knn.incremental_train(split_i, train_subsets[run_i][split_i], val_subsets[run_i][split_i])\n",
        "\n",
        "        logs[run_i].append({})\n",
        "        \n",
        "        # Test KNN classifier with only exemplars\n",
        "        acc, best_clf, targets, preds = \\\n",
        "            icarl_knn.test_knn(test_subsets[run_i][split_i], train_subsets[run_i][split_i], val_subsets[run_i][split_i], params, only_exemplars=True)\n",
        "        logs[run_i][split_i]['exemplars_accuracy'] = acc\n",
        "        logs[run_i][split_i]['exemplars_best_clf'] = best_clf\n",
        "        logs[run_i][split_i]['exemplars_conf_mat'] = confusion_matrix(targets.to('cpu'), preds.to('cpu'))\n",
        "\n",
        "        # Test KNN classifier with all available data\n",
        "        acc, best_clf, targets, preds = \\\n",
        "            icarl_knn.test_knn(test_subsets[run_i][split_i], train_subsets[run_i][split_i], val_subsets[run_i][split_i], params, only_exemplars=False)\n",
        "        logs[run_i][split_i]['all_accuracy'] = acc\n",
        "        logs[run_i][split_i]['all_best_clf'] = best_clf\n",
        "        logs[run_i][split_i]['all_conf_mat'] = confusion_matrix(targets.to('cpu'), preds.to('cpu'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qK7FWPZCZNCO",
        "colab_type": "text"
      },
      "source": [
        "### Cosine linear layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_qiJexZnDAe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from model.resnet_cifar import resnet32cosine\n",
        "from model.resnet_cifar import CosineLayer\n",
        "\n",
        "cos = nn.CosineSimilarity(dim=1, eps=1e-08)\n",
        "\n",
        "class iCaRLwithCosine(iCaRL):\n",
        "    def classify(self, batch, train_dataset=None):\n",
        "        \"\"\"Mean of exemplars with cosine similarity classifier\"\"\"\n",
        "\n",
        "        batch_features = self.extract_features(batch)\n",
        "        for i in range(batch_features.size(0)):\n",
        "            batch_features[i] = batch_features[i]/batch_features[i].norm() # Normalize sample feature representation\n",
        "        batch_features = batch_features.to(self.device) # (batch size, 64)\n",
        "\n",
        "        if self.cached_means is None:\n",
        "            print(\"Computing mean of exemplars... \", end=\"\")\n",
        "\n",
        "            self.cached_means = []\n",
        "\n",
        "            # Number of known classes\n",
        "            num_classes = len(self.exemplars)\n",
        "\n",
        "            # Compute the means of classes with all the data available,\n",
        "            # including training data which contains samples belonging to\n",
        "            # the latest 10 classes. This will remove noise from the mean\n",
        "            # estimate, improving the results.\n",
        "            if train_dataset is not None:\n",
        "                train_features_list = [[] for _ in range(10)]\n",
        "\n",
        "                for train_sample, label in train_dataset:\n",
        "                    features = self.extract_features(train_sample, batch=False, transform=self.test_transform)\n",
        "                    features = features/features.norm()\n",
        "                    train_features_list[label % 10].append(features)\n",
        "\n",
        "            # Compute means of exemplars for all known classes\n",
        "            for y in range(num_classes):\n",
        "                if (train_dataset is not None) and (y in range(num_classes-10, num_classes)):\n",
        "                    features_list = train_features_list[y % 10]\n",
        "                else:\n",
        "                    features_list = []\n",
        "\n",
        "                for exemplar in self.exemplars[y]:\n",
        "                    features = self.extract_features(exemplar, batch=False, transform=self.test_transform)\n",
        "                    features = features/features.norm() # Normalize the feature representation of the exemplar\n",
        "                    features_list.append(features)\n",
        "                \n",
        "                features_list = torch.stack(features_list)\n",
        "                class_means = features_list.mean(dim=0)\n",
        "                class_means = class_means/class_means.norm() # Normalize the class means\n",
        "\n",
        "                self.cached_means.append(class_means)\n",
        "            \n",
        "            self.cached_means = torch.stack(self.cached_means).to(self.device)\n",
        "            print(\"done\")\n",
        "\n",
        "        batch_features = batch_features.unsqueeze(0) # (1, batch_size, 64)\n",
        "        batch_features = batch_features.expand((self.cached_means.size(0), -1, -1)) # (num_classes, batch_size, 64)\n",
        "        batch_features = batch_features.transpose(0, 1) # (batch_size, num_classes, 64) to compare to means: (num_classes, 64)\n",
        "\n",
        "        preds = []\n",
        "        for i in range(batch_features.size(0)):\n",
        "            f_arg = cos(batch_features[i], self.cached_means)\n",
        "            preds.append(torch.argmax(f_arg))\n",
        "        \n",
        "        return torch.stack(preds)\n",
        "\n",
        "    def extract_features(self, sample, batch=True, transform=None):\n",
        "        assert not (batch is False and transform is None), \"if a PIL image is passed to extract_features, a transform must be defined\"\n",
        "\n",
        "        self.net.train(False)\n",
        "        if self.best_net is not None: self.best_net.train(False)\n",
        "        if self.old_net is not None: self.old_net.train(False)\n",
        "\n",
        "        if batch is False: # Treat sample as single PIL image\n",
        "            sample = transform(sample)\n",
        "            sample = sample.unsqueeze(0) # https://stackoverflow.com/a/59566009/6486336\n",
        "\n",
        "        sample = sample.to(self.device)\n",
        "\n",
        "        if self.VALIDATE:\n",
        "            features = self.best_net(sample, features=True)\n",
        "        else:\n",
        "            features = self.net(sample, features=True)\n",
        "\n",
        "        if batch is False:\n",
        "            features = features[0]\n",
        "\n",
        "        return features\n",
        "\n",
        "    def test_fc(self, test_dataset):\n",
        "        self.net.train(False)\n",
        "        if self.best_net is not None: self.best_net.train(False) # Set Network to evaluation mode\n",
        "        if self.old_net is not None: self.old_net.train(False)\n",
        "\n",
        "        self.test_dataloader = DataLoader(test_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=4)\n",
        "\n",
        "        running_corrects = 0\n",
        "        total = 0\n",
        "\n",
        "        all_preds = torch.tensor([]) # to store all predictions\n",
        "        all_preds = all_preds.type(torch.LongTensor)\n",
        "        all_targets = torch.tensor([])\n",
        "        all_targets = all_targets.type(torch.LongTensor)\n",
        "        \n",
        "        for images, labels in self.test_dataloader:\n",
        "            images = images.to(self.device)\n",
        "            labels = labels.to(self.device)\n",
        "            total += labels.size(0)\n",
        "\n",
        "            # Forward Pass\n",
        "            with torch.no_grad():\n",
        "                if self.VALIDATE:\n",
        "                    outputs = self.best_net(images)\n",
        "                else:\n",
        "                    outputs = self.net(images)\n",
        "\n",
        "            # Get predictions\n",
        "            _, preds = torch.max(outputs.data, 1)\n",
        "\n",
        "            # Update Corrects\n",
        "            running_corrects += torch.sum(preds == labels.data).data.item()\n",
        "\n",
        "            all_targets = torch.cat(\n",
        "                (all_targets.to(self.device), labels.to(self.device)), dim=0\n",
        "            )\n",
        "\n",
        "            # Append batch predictions\n",
        "            all_preds = torch.cat(\n",
        "                (all_preds.to(self.device), preds.to(self.device)), dim=0\n",
        "            )\n",
        "\n",
        "        # Calculate accuracy\n",
        "        accuracy = running_corrects / float(total)  \n",
        "\n",
        "        print(f\"Test accuracy (Cosine): {accuracy}\")\n",
        "\n",
        "        return accuracy, all_targets, all_preds\n",
        "\n",
        "    def increment_classes(self, n=10):\n",
        "        \"\"\"Add n classes in the final cosine layer.\"\"\"\n",
        "\n",
        "        in_features = self.net.fc.in_features  # size of each input sample\n",
        "        out_features = self.net.fc.out_features  # size of each output sample\n",
        "        weight = self.net.fc.weight.data\n",
        "        eta = self.net.fc.eta.data\n",
        "\n",
        "        self.net.fc = CosineLayer(in_features, out_features+n)\n",
        "        self.net.fc.weight.data[:out_features] = weight\n",
        "        self.net.fc.eta.data = eta"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dqRxXiRyxwcD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "LR = 2\n",
        "NUM_EPOCHS = 1\n",
        "MILESTONES = [49, 63]\n",
        "GAMMA = 0.2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bl2c40z9s0sA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "logs = [[] for _ in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    net = resnet32cosine()\n",
        "    icarl_cosine = iCaRLwithCosine(DEVICE, net, LR, MOMENTUM, WEIGHT_DECAY, MILESTONES, GAMMA, NUM_EPOCHS, BATCH_SIZE, train_transform, test_transform)\n",
        "\n",
        "    for split_i in range(10):\n",
        "        print(f\"## Split {split_i} of run {run_i} ##\")\n",
        "        \n",
        "        icarl_cosine.incremental_train(split_i, train_subsets[run_i][split_i], val_subsets[run_i][split_i])\n",
        "\n",
        "        logs[run_i].append({})\n",
        "        \n",
        "        # Test Cosine layer classifier (only FC)\n",
        "        acc, targets, preds = icarl_cosine.test_fc(test_subsets[run_i][split_i])\n",
        "        logs[run_i][split_i]['cosine_fc_accuracy'] = acc\n",
        "        logs[run_i][split_i]['cosine_fc_conf_mat'] = confusion_matrix(targets.to('cpu'), preds.to('cpu'))\n",
        "\n",
        "        # Test Cosine similarity\n",
        "        acc, targets, preds = icarl_cosine.test(test_subsets[run_i][split_i], train_subsets[run_i][split_i])\n",
        "        logs[run_i][split_i]['cosine_sim_accuracy'] = acc\n",
        "        logs[run_i][split_i]['cosine_sim_conf_mat'] = confusion_matrix(targets.to('cpu'), preds.to('cpu'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3mEc4EL0BdRE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "obj_save(logs, 'cosine')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1KjCTWapgGGs",
        "colab_type": "text"
      },
      "source": [
        "### Random forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "n0MiIFXbgf7m",
        "colab": {}
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.model_selection  import ParameterGrid\n",
        "from copy import deepcopy\n",
        "\n",
        "class iCaRLwithRF(iCaRL):\n",
        "    def classifier_fit(self, train_dataset, val_dataset, params, only_exemplars=False):\n",
        "        \"\"\"Fit classifier on the union of training dataset and exemplars,\n",
        "        and validate it on val_dataset.\"\"\"\n",
        "\n",
        "        if only_exemplars:\n",
        "            fit_dataset = Exemplars(self.exemplars, self.train_transform)\n",
        "        else:\n",
        "            # Union of training dataset and exemplars\n",
        "            exemplars_dataset = Exemplars(self.exemplars, self.train_transform)\n",
        "            fit_dataset = ConcatDataset([exemplars_dataset, train_dataset])\n",
        "\n",
        "        # Convert dataset to numpy format\n",
        "        # X contains training samples, y contains labels\n",
        "        X, y = self.dataset_to_numpy(fit_dataset)\n",
        "\n",
        "        # Extract features from the training dataset\n",
        "        X_features = self.extract_features(torch.tensor(X, dtype=torch.float))\n",
        "        for i in range(X_features.size(0)):\n",
        "            X_features[i] = X_features[i]/X_features[i].norm()\n",
        "        X_features = X_features.to('cpu').numpy()\n",
        "\n",
        "        # Initialize classifier\n",
        "        self.clf = RandomForestClassifier()\n",
        "\n",
        "        # Run validation\n",
        "        best_clf = None\n",
        "        best_grid = None\n",
        "        best_score = 0\n",
        "\n",
        "        X_test, y_test = self.dataset_to_numpy(val_dataset)\n",
        "        X_test_features = self.extract_features(torch.tensor(X_test, dtype=torch.float))\n",
        "        for i in range(X_test_features.size(0)):\n",
        "            X_test_features[i] = X_test_features[i]/X_test_features[i].norm()\n",
        "        X_test_features = X_test_features.to('cpu').numpy()\n",
        "\n",
        "        for grid in ParameterGrid(params):\n",
        "            self.clf.set_params(**grid)\n",
        "            self.clf.fit(X_features, y)\n",
        "            y_pred = self.clf.predict(X_test_features)\n",
        "            score = accuracy_score(y_test, y_pred)\n",
        "\n",
        "            if score > best_score:\n",
        "                best_clf = deepcopy(self.clf)\n",
        "                best_score = score\n",
        "                best_grid = grid\n",
        "\n",
        "        # Set the classifier to the best clf found in validation\n",
        "        self.clf = best_clf\n",
        "\n",
        "        print(f\"Best classifier: {best_grid} with score {best_score}\")\n",
        "\n",
        "        return best_grid\n",
        "\n",
        "    def classifier_predict(self, test_dataset):\n",
        "        \"\"\"Predict labels of test_dataset.\"\"\"\n",
        "\n",
        "        X_test, y_test = self.dataset_to_numpy(test_dataset)\n",
        "\n",
        "        # Extract features from the test set\n",
        "        X_test_features = self.extract_features(torch.tensor(X_test, dtype=torch.float))\n",
        "        for i in range(X_test_features.size(0)):\n",
        "            X_test_features[i] = X_test_features[i]/X_test_features[i].norm()\n",
        "        X_test_features = X_test_features.to('cpu').numpy()\n",
        "        \n",
        "        y_pred = self.clf.predict(X_test_features)\n",
        "\n",
        "        return y_test, y_pred\n",
        "\n",
        "    def dataset_to_numpy(self, dataset):\n",
        "        # Preallocate arrays\n",
        "        X = np.zeros((len(dataset), 3, 32, 32))\n",
        "        y = np.zeros(len(dataset), dtype=int)\n",
        "\n",
        "        dataloader = DataLoader(dataset, batch_size=1)\n",
        "\n",
        "        for idx, (image, labels) in enumerate(dataloader):\n",
        "            X[idx] = image[0].numpy()\n",
        "            y[idx] = labels.numpy()[0]\n",
        "\n",
        "        return X, y\n",
        "\n",
        "    def test_rf(self, test_dataset, train_dataset, val_dataset, params, only_exemplars=False):\n",
        "        \"\"\"Test the model.\n",
        "\n",
        "        Args:\n",
        "            test_dataset: dataset on which to test the network\n",
        "            train_dataset: training set used to train the last split\n",
        "            params: parameter grid on which to perform hyperparameter tuning\n",
        "        Returns:\n",
        "            accuracy (float): accuracy of the model on the test set\n",
        "        \"\"\"\n",
        "\n",
        "        self.net.train(False)\n",
        "        if self.best_net is not None: self.best_net.train(False)  # Set Network to evaluation mode\n",
        "        if self.old_net is not None: self.old_net.train(False)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            # Use val_dataset as validation set for hyperparameter tuning.\n",
        "            best_grid = self.classifier_fit(train_dataset, val_dataset, params, only_exemplars)\n",
        "\n",
        "            y_truth, y_pred = self.classifier_predict(test_dataset)\n",
        "            accuracy = accuracy_score(y_truth, y_pred)\n",
        "\n",
        "            if only_exemplars:\n",
        "                print(f\"Test accuracy (iCaRL with RF only exemplars): {accuracy} \")\n",
        "            else:\n",
        "                print(f\"Test accuracy (iCaRL with RF all available data): {accuracy} \")\n",
        "\n",
        "        return accuracy, best_grid, torch.tensor(y_truth), torch.tensor(y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Q_4pkwQogf7s",
        "colab": {}
      },
      "source": [
        "NUM_EPOCHS = 70"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0f7bhOpNgf7x",
        "colab": {}
      },
      "source": [
        "logs = [[] for _ in range(NUM_RUNS)]\n",
        "\n",
        "params = {\n",
        "    \"n_estimators\": [100, 200, 500, 1000],\n",
        "    \"min_samples_split\": [10, 20, 50]\n",
        "}\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    net = resnet32()\n",
        "    icarl_rf = iCaRLwithRF(DEVICE, net, LR, MOMENTUM, WEIGHT_DECAY, MILESTONES, GAMMA, NUM_EPOCHS, BATCH_SIZE, train_transform, test_transform)\n",
        "\n",
        "    for split_i in range(10):\n",
        "        print(f\"## Split {split_i} of run {run_i} ##\")\n",
        "        \n",
        "        icarl_rf.incremental_train(split_i, train_subsets[run_i][split_i], val_subsets[run_i][split_i])\n",
        "\n",
        "        logs[run_i].append({})\n",
        "        \n",
        "        # Test RF classifier with only exemplars\n",
        "        acc, best_clf, targets, preds = \\\n",
        "            icarl_rf.test_rf(test_subsets[run_i][split_i], train_subsets[run_i][split_i], val_subsets[run_i][split_i], params, only_exemplars=True)\n",
        "        logs[run_i][split_i]['exemplars_accuracy'] = acc\n",
        "        logs[run_i][split_i]['exemplars_best_clf'] = best_clf\n",
        "        logs[run_i][split_i]['exemplars_conf_mat'] = confusion_matrix(targets.to('cpu'), preds.to('cpu'))\n",
        "\n",
        "        # Test RF classifier with all available data\n",
        "        acc, best_clf, targets, preds = \\\n",
        "            icarl_rf.test_rf(test_subsets[run_i][split_i], train_subsets[run_i][split_i], val_subsets[run_i][split_i], params, only_exemplars=False)\n",
        "        logs[run_i][split_i]['all_accuracy'] = acc\n",
        "        logs[run_i][split_i]['all_best_clf'] = best_clf\n",
        "        logs[run_i][split_i]['all_conf_mat'] = confusion_matrix(targets.to('cpu'), preds.to('cpu'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gQN43l8LhWsB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "obj_save(logs, 'rf_logs')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}