GlioML-Main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa55bed2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import collections\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from autogluon.tabular import TabularDataset, TabularPredictor\n",
    "from autogluon.common.utils.utils import setup_outputdir\n",
    "from autogluon.core.utils.loaders import load_pkl\n",
    "from autogluon.core.utils.savers import save_pkl\n",
    "import os.path\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from random import sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "41b72ee4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultilabelPredictor():\n",
    "    \"\"\" Tabular Predictor for predicting multiple columns in table.\n",
    "        Creates multiple TabularPredictor objects which you can also use individually.\n",
    "        You can access the TabularPredictor for a particular label via: `multilabel_predictor.get_predictor(label_i)`\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        labels : List[str]\n",
    "            The ith element of this list is the column (i.e. `label`) predicted by the ith TabularPredictor stored in this object.\n",
    "        path : str, default = None\n",
    "            Path to directory where models and intermediate outputs should be saved.\n",
    "            If unspecified, a time-stamped folder called \"AutogluonModels/ag-[TIMESTAMP]\" will be created in the working directory to store all models.\n",
    "            Note: To call `fit()` twice and save all results of each fit, you must specify different `path` locations or don't specify `path` at all.\n",
    "            Otherwise files from first `fit()` will be overwritten by second `fit()`.\n",
    "            Caution: when predicting many labels, this directory may grow large as it needs to store many TabularPredictors.\n",
    "        problem_types : List[str], default = None\n",
    "            The ith element is the `problem_type` for the ith TabularPredictor stored in this object.\n",
    "        eval_metrics : List[str], default = None\n",
    "            The ith element is the `eval_metric` for the ith TabularPredictor stored in this object.\n",
    "        consider_labels_correlation : bool, default = True\n",
    "            Whether the predictions of multiple labels should account for label correlations or predict each label independently of the others.\n",
    "            If True, the ordering of `labels` may affect resulting accuracy as each label is predicted conditional on the previous labels appearing earlier in this list (i.e. in an auto-regressive fashion).\n",
    "            Set to False if during inference you may want to individually use just the ith TabularPredictor without predicting all the other labels.\n",
    "        kwargs :\n",
    "            Arguments passed into the initialization of each TabularPredictor.\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    multi_predictor_file = 'multilabel_predictor.pkl'\n",
    "\n",
    "    def __init__(self, labels, path=None, problem_types=None, eval_metrics=None, consider_labels_correlation=False, **kwargs):\n",
    "        if (problem_types is not None) and (len(problem_types) != len(labels)):\n",
    "            raise ValueError(\"If provided, `problem_types` must have same length as `labels`\")\n",
    "        if (eval_metrics is not None) and (len(eval_metrics) != len(labels)):\n",
    "            raise ValueError(\"If provided, `eval_metrics` must have same length as `labels`\")\n",
    "        self.path = setup_outputdir(path, warn_if_exist=False)\n",
    "        self.labels = labels\n",
    "        self.consider_labels_correlation = consider_labels_correlation\n",
    "        self.predictors = {}  # key = label, value = TabularPredictor or str path to the TabularPredictor for this label\n",
    "        if eval_metrics is None:\n",
    "            self.eval_metrics = {}\n",
    "        else:\n",
    "            self.eval_metrics = {labels[i] : eval_metrics[i] for i in range(len(labels))}\n",
    "        problem_type = None\n",
    "        eval_metric = None\n",
    "        for i in range(len(labels)):\n",
    "            label = labels[i]\n",
    "            path_i = self.path + \"Predictor_\" + label\n",
    "            if problem_types is not None:\n",
    "                problem_type = problem_types[i]\n",
    "            if eval_metrics is not None:\n",
    "                eval_metric = eval_metrics[i]\n",
    "            self.predictors[label] = TabularPredictor(label=label, problem_type=problem_type, eval_metric=eval_metric, path=path_i, **kwargs)\n",
    "\n",
    "    def fit(self, train_data, tuning_data=None, final = False, **kwargs):\n",
    "        \"\"\" Fits a separate TabularPredictor to predict each of the labels.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            train_data, tuning_data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                See documentation for `TabularPredictor.fit()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `fit()` call for each TabularPredictor.\n",
    "        \"\"\"\n",
    "        if isinstance(train_data, str):\n",
    "            train_data = TabularDataset(train_data)\n",
    "        if tuning_data is not None and isinstance(tuning_data, str):\n",
    "            tuning_data = TabularDataset(tuning_data)\n",
    "        train_data_og = train_data.copy()\n",
    "        if tuning_data is not None:\n",
    "            tuning_data_og = tuning_data.copy()\n",
    "        else:\n",
    "            tuning_data_og = None\n",
    "        save_metrics = len(self.eval_metrics) == 0\n",
    "        start = time.time()\n",
    "        for i in range(len(self.labels)):\n",
    "            label = self.labels[i]\n",
    "            predictor = self.get_predictor(label)\n",
    "            if not self.consider_labels_correlation:\n",
    "                labels_to_drop = [l for l in self.labels if l != label]\n",
    "            else:\n",
    "                labels_to_drop = [self.labels[j] for j in range(i+1, len(self.labels))]\n",
    "            train_data = train_data_og.drop(labels_to_drop, axis=1)\n",
    "            if tuning_data is not None:\n",
    "                tuning_data = tuning_data_og.drop(labels_to_drop, axis=1)\n",
    "            print(f\"Fitting TabularPredictor for label: {label} ...{i / len(self.labels) * 100}%\")\n",
    "            print(f\"{(time.time() - start) / 60} minutes\")\n",
    "            if (final):\n",
    "                predictor.fit(train_data=train_data[train_data[label] > float('-inf')]\n",
    "                              , tuning_data = tuning_data\n",
    "                              ,presets = 'best_quality'\n",
    "                              ,num_bag_folds = 5,num_bag_sets = 2\n",
    "                              , **kwargs)\n",
    "            else:\n",
    "                predictor.fit(train_data=train_data[train_data[label] > float('-inf')]\n",
    "                              , tuning_data = tuning_data\n",
    "                              ,presets = 'medium_quality'\n",
    "                              #,presets = 'best_quality'\n",
    "                              #,num_bag_folds = 5,num_bag_sets = 2\n",
    "                              , **kwargs)\n",
    "            self.predictors[label] = predictor.path\n",
    "            if save_metrics:\n",
    "                self.eval_metrics[label] = predictor.eval_metric\n",
    "        self.save()\n",
    "\n",
    "    def predict(self, data, **kwargs):\n",
    "        \"\"\" Returns DataFrame with label columns containing predictions for each label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to make predictions for. If label columns are present in this data, they will be ignored. See documentation for `TabularPredictor.predict()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the predict() call for each TabularPredictor.\n",
    "        \"\"\"\n",
    "        return self._predict(data, as_proba=False, **kwargs)\n",
    "\n",
    "    def predict_proba(self, data, **kwargs):\n",
    "        \"\"\" Returns dict where each key is a label and the corresponding value is the `predict_proba()` output for just that label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to make predictions for. See documentation for `TabularPredictor.predict()` and `TabularPredictor.predict_proba()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `predict_proba()` call for each TabularPredictor (also passed into a `predict()` call).\n",
    "        \"\"\"\n",
    "        return self._predict(data, as_proba=True, **kwargs)\n",
    "\n",
    "    def evaluate(self, data, **kwargs):\n",
    "        \"\"\" Returns dict where each key is a label and the corresponding value is the `evaluate()` output for just that label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to evalate predictions of all labels for, must contain all labels as columns. See documentation for `TabularPredictor.evaluate()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `evaluate()` call for each TabularPredictor (also passed into the `predict()` call).\n",
    "        \"\"\"\n",
    "        data = self._get_data(data)\n",
    "        eval_dict = {}\n",
    "        for label in self.labels:\n",
    "            print(f\"Evaluating TabularPredictor for label: {label} ...\")\n",
    "            predictor = self.get_predictor(label)\n",
    "            \n",
    "            eval_dict[label] = predictor.evaluate(data[data[label] > float('-inf')], **kwargs)\n",
    "            if self.consider_labels_correlation:\n",
    "                data[label] = predictor.predict(data, **kwargs)\n",
    "        return eval_dict\n",
    "\n",
    "    def save(self):\n",
    "        \"\"\" Save MultilabelPredictor to disk. \"\"\"\n",
    "        for label in self.labels:\n",
    "            if not isinstance(self.predictors[label], str):\n",
    "                self.predictors[label] = self.predictors[label].path\n",
    "        save_pkl.save(path=self.path+self.multi_predictor_file, object=self)\n",
    "        print(f\"MultilabelPredictor saved to disk. Load with: MultilabelPredictor.load('{self.path}')\")\n",
    "\n",
    "    @classmethod\n",
    "    def load(cls, path):\n",
    "        \"\"\" Load MultilabelPredictor from disk `path` previously specified when creating this MultilabelPredictor. \"\"\"\n",
    "        path = os.path.expanduser(path)\n",
    "        if path[-1] != os.path.sep:\n",
    "            path = path + os.path.sep\n",
    "        return load_pkl.load(path=path+cls.multi_predictor_file)\n",
    "\n",
    "    def get_predictor(self, label):\n",
    "        \"\"\" Returns TabularPredictor which is used to predict this label. \"\"\"\n",
    "        predictor = self.predictors[label]\n",
    "        if isinstance(predictor, str):\n",
    "            return TabularPredictor.load(path=predictor)\n",
    "        return predictor\n",
    "\n",
    "    def _get_data(self, data):\n",
    "        if isinstance(data, str):\n",
    "            return TabularDataset(data)\n",
    "        return data.copy()\n",
    "\n",
    "    def _predict(self, data, as_proba=False, **kwargs):\n",
    "        data = self._get_data(data)\n",
    "        if as_proba:\n",
    "            predproba_dict = {}\n",
    "        for i,label in enumerate(self.labels):\n",
    "            print(f\"Predicting with TabularPredictor for label: {label} ...{i / len(self.labels) * 100}%\")\n",
    "            predictor = self.get_predictor(label)\n",
    "            if as_proba:\n",
    "                predproba_dict[label] = predictor.predict_proba(data, as_multiclass=True, **kwargs)\n",
    "            data[label] = predictor.predict(data, **kwargs)\n",
    "        if not as_proba:\n",
    "            return data[self.labels]\n",
    "        else:\n",
    "            return predproba_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d3ae94b9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-02T06:09:54.216156Z",
     "start_time": "2022-07-02T06:09:54.210855Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293\n",
      "187\n",
      "197\n",
      "1616\n",
      "665\n",
      "280\n",
      "53\n"
     ]
    }
   ],
   "source": [
    "#1 Load my ccl's ssGSEA signature\n",
    "myCCLSignature = []\n",
    "for name in ['sample.c2.cp.biocarta.gct',\n",
    "             'sample.c2.cp.kegg.gct',\n",
    "             'sample.c2.cp.pid.gct',\n",
    "             'sample.c2.cp.reactome.gct',\n",
    "             'sample.c2.cp.wiki.gct',\n",
    "             'sample.c6.gct',\n",
    "             'sample.hallmark.gct']:    \n",
    "    with open(name, mode ='r')as file:\n",
    "        csvFile = csv.reader(file)\n",
    "        CCLSignature = list(csvFile)[2:]\n",
    "        print(len(CCLSignature))\n",
    "    for i, row in enumerate(CCLSignature):\n",
    "        temp = CCLSignature[i][0].split('\\t')\n",
    "        if i > 0:\n",
    "            CCLSignature[i] = [temp[0]] + [float(d) for d in temp[2:]]\n",
    "        else:\n",
    "            CCLSignature[i] = [temp[0]] + temp[2:]\n",
    "    if not myCCLSignature:\n",
    "        myCCLSignature += CCLSignature\n",
    "    else:\n",
    "        myCCLSignature += CCLSignature[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a1ebef18",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-02T06:10:01.747913Z",
     "start_time": "2022-07-02T06:10:01.683560Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293\n",
      "187\n",
      "197\n",
      "1616\n",
      "665\n",
      "280\n",
      "53\n"
     ]
    }
   ],
   "source": [
    "#2 Load CCLE ssGSEA signature\n",
    "CCLECCLSignature = []\n",
    "for name in ['ccle.c2.cp.biocarta.gct',\n",
    "             'ccle.c2.cp.kegg.gct',\n",
    "             'ccle.c2.cp.pid.gct',\n",
    "             'ccle.c2.cp.reactome.gct',\n",
    "             'ccle.c2.cp.wiki.gct',\n",
    "             'ccle.c6.gct',\n",
    "             'ccle.hallmark.gct']:\n",
    "    with open(name, mode ='r')as file:\n",
    "        csvFile = csv.reader(file)\n",
    "        CCLSignature = list(csvFile)[2:]\n",
    "        print(len(CCLSignature))\n",
    "    for i, row in enumerate(CCLSignature):\n",
    "        temp = CCLSignature[i][0].split('\\t')\n",
    "        CCLSignature[i] = [temp[0]] + temp[2:]\n",
    "    if not CCLECCLSignature:\n",
    "        CCLECCLSignature += CCLSignature\n",
    "    else:\n",
    "        CCLECCLSignature += CCLSignature[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ce386d0b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-02T06:10:47.896499Z",
     "start_time": "2022-07-02T06:10:47.530996Z"
    }
   },
   "outputs": [],
   "source": [
    "#3 Load CTRP cclName to AUC map\n",
    "cclToAUCdict = collections.defaultdict(list)\n",
    "with open('CTRP_CCL_AUC.gct', mode ='r') as file:\n",
    "    csvFile = csv.reader(file)\n",
    "    CTRPCCLAUC = list(csvFile)\n",
    "    CTRPCCLAUC = [''.join(sub).split('\\t') for sub in CTRPCCLAUC]\n",
    "    cclNames = CTRPCCLAUC[3][4:]\n",
    "\n",
    "for i,cclName in enumerate(cclNames):\n",
    "    cclToAUCdict[cclName] = [float( '-inf' if sub[4+i] == 'NaN' else sub[4+i]) for sub in CTRPCCLAUC[7:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dfc80f9d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-02T06:10:57.926109Z",
     "start_time": "2022-07-02T06:10:57.915764Z"
    }
   },
   "outputs": [],
   "source": [
    "#4 Load ccleID to ctrpName map\n",
    "CCLEidToCTRPNameDict = collections.defaultdict(str)\n",
    "CCLEidToDiseaseName = collections.defaultdict(str)\n",
    "with open('sample_info.csv', mode ='r') as file:\n",
    "    csvFile = csv.reader(file)\n",
    "    mapInfos = list(csvFile)\n",
    "    for mapInfo in mapInfos[1:]:\n",
    "        CCLEidToCTRPNameDict[mapInfo[0]] = mapInfo[2]  \n",
    "        CCLEidToDiseaseName[mapInfo[0]] = mapInfo[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8996ddbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(df):\n",
    "    result = df.copy()\n",
    "    for feature_name in df.columns:\n",
    "        max_value = df[feature_name].max()\n",
    "        min_value = df[feature_name].min()\n",
    "        result[feature_name] = (df[feature_name] - min_value) / (max_value - min_value)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cc53a03e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIOCARTA_GRANULOCYTES_PATHWAY</th>\n",
       "      <th>BIOCARTA_LYM_PATHWAY</th>\n",
       "      <th>BIOCARTA_BLYMPHOCYTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_CARM_ER_PATHWAY</th>\n",
       "      <th>BIOCARTA_LAIR_PATHWAY</th>\n",
       "      <th>BIOCARTA_VDR_PATHWAY</th>\n",
       "      <th>BIOCARTA_MTA3_PATHWAY</th>\n",
       "      <th>BIOCARTA_GABA_PATHWAY</th>\n",
       "      <th>BIOCARTA_EGFR_SMRTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_MONOCYTE_PATHWAY</th>\n",
       "      <th>...</th>\n",
       "      <th>HALLMARK_COAGULATION</th>\n",
       "      <th>HALLMARK_IL2_STAT5_SIGNALING</th>\n",
       "      <th>HALLMARK_BILE_ACID_METABOLISM</th>\n",
       "      <th>HALLMARK_PEROXISOME</th>\n",
       "      <th>HALLMARK_ALLOGRAFT_REJECTION</th>\n",
       "      <th>HALLMARK_SPERMATOGENESIS</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_UP</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_DN</th>\n",
       "      <th>HALLMARK_PANCREAS_BETA_CELLS</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>THP1_d3_B.TPM</th>\n",
       "      <td>0.991211</td>\n",
       "      <td>0.985352</td>\n",
       "      <td>0.984375</td>\n",
       "      <td>0.876465</td>\n",
       "      <td>0.993652</td>\n",
       "      <td>0.595703</td>\n",
       "      <td>0.475098</td>\n",
       "      <td>0.492432</td>\n",
       "      <td>0.594238</td>\n",
       "      <td>0.986328</td>\n",
       "      <td>...</td>\n",
       "      <td>0.675781</td>\n",
       "      <td>0.810059</td>\n",
       "      <td>0.680176</td>\n",
       "      <td>0.774902</td>\n",
       "      <td>0.928711</td>\n",
       "      <td>0.793457</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.880859</td>\n",
       "      <td>0.337402</td>\n",
       "      <td>0.670410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2D_1.TPM</th>\n",
       "      <td>0.160278</td>\n",
       "      <td>0.161499</td>\n",
       "      <td>0.059387</td>\n",
       "      <td>0.379395</td>\n",
       "      <td>0.045349</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.089844</td>\n",
       "      <td>0.554199</td>\n",
       "      <td>0.363525</td>\n",
       "      <td>0.187866</td>\n",
       "      <td>...</td>\n",
       "      <td>0.025284</td>\n",
       "      <td>0.065186</td>\n",
       "      <td>0.221802</td>\n",
       "      <td>0.194458</td>\n",
       "      <td>0.019119</td>\n",
       "      <td>0.555664</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.037323</td>\n",
       "      <td>0.343262</td>\n",
       "      <td>0.248169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TRId7CWa.quant.TPM</th>\n",
       "      <td>0.438232</td>\n",
       "      <td>0.443604</td>\n",
       "      <td>0.540527</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.506836</td>\n",
       "      <td>0.303955</td>\n",
       "      <td>0.583496</td>\n",
       "      <td>0.429932</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700195</td>\n",
       "      <td>...</td>\n",
       "      <td>0.927246</td>\n",
       "      <td>0.736328</td>\n",
       "      <td>0.484619</td>\n",
       "      <td>0.130737</td>\n",
       "      <td>0.436523</td>\n",
       "      <td>0.053558</td>\n",
       "      <td>0.354004</td>\n",
       "      <td>0.771484</td>\n",
       "      <td>0.968750</td>\n",
       "      <td>0.720703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>H7.TPM</th>\n",
       "      <td>0.553711</td>\n",
       "      <td>0.562500</td>\n",
       "      <td>0.427979</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.724609</td>\n",
       "      <td>0.403809</td>\n",
       "      <td>0.853027</td>\n",
       "      <td>0.493896</td>\n",
       "      <td>0.511230</td>\n",
       "      <td>0.531738</td>\n",
       "      <td>...</td>\n",
       "      <td>0.822754</td>\n",
       "      <td>0.816406</td>\n",
       "      <td>0.113708</td>\n",
       "      <td>0.248291</td>\n",
       "      <td>0.477051</td>\n",
       "      <td>0.189209</td>\n",
       "      <td>0.544434</td>\n",
       "      <td>0.707520</td>\n",
       "      <td>0.644043</td>\n",
       "      <td>0.258057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>THP1_d3_A.TPM</th>\n",
       "      <td>0.992676</td>\n",
       "      <td>0.992676</td>\n",
       "      <td>0.973633</td>\n",
       "      <td>0.795898</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.548340</td>\n",
       "      <td>0.615723</td>\n",
       "      <td>0.447266</td>\n",
       "      <td>0.474854</td>\n",
       "      <td>0.989258</td>\n",
       "      <td>...</td>\n",
       "      <td>0.691406</td>\n",
       "      <td>0.826660</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>0.877930</td>\n",
       "      <td>0.928223</td>\n",
       "      <td>0.791504</td>\n",
       "      <td>0.947754</td>\n",
       "      <td>0.933105</td>\n",
       "      <td>0.476562</td>\n",
       "      <td>0.819824</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3284 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0                   BIOCARTA_GRANULOCYTES_PATHWAY  BIOCARTA_LYM_PATHWAY  \\\n",
       "Name                                                                      \n",
       "THP1_d3_B.TPM                            0.991211              0.985352   \n",
       "2D_1.TPM                                 0.160278              0.161499   \n",
       "TRId7CWa.quant.TPM                       0.438232              0.443604   \n",
       "H7.TPM                                   0.553711              0.562500   \n",
       "THP1_d3_A.TPM                            0.992676              0.992676   \n",
       "\n",
       "0                   BIOCARTA_BLYMPHOCYTE_PATHWAY  BIOCARTA_CARM_ER_PATHWAY  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                           0.984375                  0.876465   \n",
       "2D_1.TPM                                0.059387                  0.379395   \n",
       "TRId7CWa.quant.TPM                      0.540527                  0.000000   \n",
       "H7.TPM                                  0.427979                  1.000000   \n",
       "THP1_d3_A.TPM                           0.973633                  0.795898   \n",
       "\n",
       "0                   BIOCARTA_LAIR_PATHWAY  BIOCARTA_VDR_PATHWAY  \\\n",
       "Name                                                              \n",
       "THP1_d3_B.TPM                    0.993652              0.595703   \n",
       "2D_1.TPM                         0.045349              1.000000   \n",
       "TRId7CWa.quant.TPM               0.506836              0.303955   \n",
       "H7.TPM                           0.724609              0.403809   \n",
       "THP1_d3_A.TPM                    1.000000              0.548340   \n",
       "\n",
       "0                   BIOCARTA_MTA3_PATHWAY  BIOCARTA_GABA_PATHWAY  \\\n",
       "Name                                                               \n",
       "THP1_d3_B.TPM                    0.475098               0.492432   \n",
       "2D_1.TPM                         0.089844               0.554199   \n",
       "TRId7CWa.quant.TPM               0.583496               0.429932   \n",
       "H7.TPM                           0.853027               0.493896   \n",
       "THP1_d3_A.TPM                    0.615723               0.447266   \n",
       "\n",
       "0                   BIOCARTA_EGFR_SMRTE_PATHWAY  BIOCARTA_MONOCYTE_PATHWAY  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                          0.594238                   0.986328   \n",
       "2D_1.TPM                               0.363525                   0.187866   \n",
       "TRId7CWa.quant.TPM                     1.000000                   0.700195   \n",
       "H7.TPM                                 0.511230                   0.531738   \n",
       "THP1_d3_A.TPM                          0.474854                   0.989258   \n",
       "\n",
       "0                   ...  HALLMARK_COAGULATION  HALLMARK_IL2_STAT5_SIGNALING  \\\n",
       "Name                ...                                                       \n",
       "THP1_d3_B.TPM       ...              0.675781                      0.810059   \n",
       "2D_1.TPM            ...              0.025284                      0.065186   \n",
       "TRId7CWa.quant.TPM  ...              0.927246                      0.736328   \n",
       "H7.TPM              ...              0.822754                      0.816406   \n",
       "THP1_d3_A.TPM       ...              0.691406                      0.826660   \n",
       "\n",
       "0                   HALLMARK_BILE_ACID_METABOLISM  HALLMARK_PEROXISOME  \\\n",
       "Name                                                                     \n",
       "THP1_d3_B.TPM                            0.680176             0.774902   \n",
       "2D_1.TPM                                 0.221802             0.194458   \n",
       "TRId7CWa.quant.TPM                       0.484619             0.130737   \n",
       "H7.TPM                                   0.113708             0.248291   \n",
       "THP1_d3_A.TPM                            0.710938             0.877930   \n",
       "\n",
       "0                   HALLMARK_ALLOGRAFT_REJECTION  HALLMARK_SPERMATOGENESIS  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                           0.928711                  0.793457   \n",
       "2D_1.TPM                                0.019119                  0.555664   \n",
       "TRId7CWa.quant.TPM                      0.436523                  0.053558   \n",
       "H7.TPM                                  0.477051                  0.189209   \n",
       "THP1_d3_A.TPM                           0.928223                  0.791504   \n",
       "\n",
       "0                   HALLMARK_KRAS_SIGNALING  HALLMARK_KRAS_SIGNALING_UP  \\\n",
       "Name                                                                      \n",
       "THP1_d3_B.TPM                      1.000000                    0.880859   \n",
       "2D_1.TPM                           0.000000                    0.037323   \n",
       "TRId7CWa.quant.TPM                 0.354004                    0.771484   \n",
       "H7.TPM                             0.544434                    0.707520   \n",
       "THP1_d3_A.TPM                      0.947754                    0.933105   \n",
       "\n",
       "0                   HALLMARK_KRAS_SIGNALING_DN  HALLMARK_PANCREAS_BETA_CELLS  \n",
       "Name                                                                          \n",
       "THP1_d3_B.TPM                         0.337402                      0.670410  \n",
       "2D_1.TPM                              0.343262                      0.248169  \n",
       "TRId7CWa.quant.TPM                    0.968750                      0.720703  \n",
       "H7.TPM                                0.644043                      0.258057  \n",
       "THP1_d3_A.TPM                         0.476562                      0.819824  \n",
       "\n",
       "[5 rows x 3284 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prediction data \n",
    "predictData = pd.DataFrame(data = myCCLSignature).transpose()\n",
    "new_header = predictData.iloc[0] \n",
    "predictData = predictData[1:] \n",
    "predictData.columns = new_header \n",
    "predictData = predictData.apply(pd.to_numeric, errors='ignore')\n",
    "predictData = predictData.set_index(['Name'])\n",
    "predictData = normalize(predictData)\n",
    "predictData = predictData.astype('float16')\n",
    "predictData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "06a614de",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 26 entries, THP1_d3_B.TPM to HCd7CWa.quant.TPM\n",
      "Columns: 3284 entries, BIOCARTA_GRANULOCYTES_PATHWAY to HALLMARK_PANCREAS_BETA_CELLS\n",
      "dtypes: float16(3284)\n",
      "memory usage: 168.5 KB\n"
     ]
    }
   ],
   "source": [
    "predictData.info(memory_usage = 'deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c6935a1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare train set\n",
    "trainData = pd.DataFrame(data = CCLECCLSignature).transpose()\n",
    "new_header = trainData.iloc[0] \n",
    "trainData = trainData[1:] \n",
    "trainData.columns = new_header "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "52014506",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIOCARTA_GRANULOCYTES_PATHWAY</th>\n",
       "      <th>BIOCARTA_LYM_PATHWAY</th>\n",
       "      <th>BIOCARTA_BLYMPHOCYTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_CARM_ER_PATHWAY</th>\n",
       "      <th>BIOCARTA_LAIR_PATHWAY</th>\n",
       "      <th>BIOCARTA_VDR_PATHWAY</th>\n",
       "      <th>BIOCARTA_MTA3_PATHWAY</th>\n",
       "      <th>BIOCARTA_GABA_PATHWAY</th>\n",
       "      <th>BIOCARTA_EGFR_SMRTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_MONOCYTE_PATHWAY</th>\n",
       "      <th>...</th>\n",
       "      <th>HALLMARK_COAGULATION</th>\n",
       "      <th>HALLMARK_IL2_STAT5_SIGNALING</th>\n",
       "      <th>HALLMARK_BILE_ACID_METABOLISM</th>\n",
       "      <th>HALLMARK_PEROXISOME</th>\n",
       "      <th>HALLMARK_ALLOGRAFT_REJECTION</th>\n",
       "      <th>HALLMARK_SPERMATOGENESIS</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_UP</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_DN</th>\n",
       "      <th>HALLMARK_PANCREAS_BETA_CELLS</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ACH-001113</th>\n",
       "      <td>0.201904</td>\n",
       "      <td>0.214722</td>\n",
       "      <td>0.107483</td>\n",
       "      <td>0.436279</td>\n",
       "      <td>0.251221</td>\n",
       "      <td>0.543945</td>\n",
       "      <td>0.402344</td>\n",
       "      <td>0.446289</td>\n",
       "      <td>0.692383</td>\n",
       "      <td>0.299561</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369873</td>\n",
       "      <td>0.589355</td>\n",
       "      <td>0.231079</td>\n",
       "      <td>0.508301</td>\n",
       "      <td>0.178223</td>\n",
       "      <td>0.653809</td>\n",
       "      <td>0.345215</td>\n",
       "      <td>0.352539</td>\n",
       "      <td>0.390381</td>\n",
       "      <td>0.233032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000242</th>\n",
       "      <td>0.210693</td>\n",
       "      <td>0.195068</td>\n",
       "      <td>0.137573</td>\n",
       "      <td>0.598633</td>\n",
       "      <td>0.192383</td>\n",
       "      <td>0.888672</td>\n",
       "      <td>0.381348</td>\n",
       "      <td>0.108154</td>\n",
       "      <td>0.679688</td>\n",
       "      <td>0.285400</td>\n",
       "      <td>...</td>\n",
       "      <td>0.448730</td>\n",
       "      <td>0.651367</td>\n",
       "      <td>0.461914</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>0.337402</td>\n",
       "      <td>0.380371</td>\n",
       "      <td>0.503906</td>\n",
       "      <td>0.462646</td>\n",
       "      <td>0.288818</td>\n",
       "      <td>0.128052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000327</th>\n",
       "      <td>0.226074</td>\n",
       "      <td>0.228394</td>\n",
       "      <td>0.131714</td>\n",
       "      <td>0.807617</td>\n",
       "      <td>0.214966</td>\n",
       "      <td>0.105347</td>\n",
       "      <td>0.366455</td>\n",
       "      <td>0.534668</td>\n",
       "      <td>0.508789</td>\n",
       "      <td>0.279541</td>\n",
       "      <td>...</td>\n",
       "      <td>0.377441</td>\n",
       "      <td>0.626953</td>\n",
       "      <td>0.639648</td>\n",
       "      <td>0.951172</td>\n",
       "      <td>0.237305</td>\n",
       "      <td>0.491943</td>\n",
       "      <td>0.441162</td>\n",
       "      <td>0.435791</td>\n",
       "      <td>0.353760</td>\n",
       "      <td>0.339844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000461</th>\n",
       "      <td>0.230469</td>\n",
       "      <td>0.359375</td>\n",
       "      <td>0.087646</td>\n",
       "      <td>0.528809</td>\n",
       "      <td>0.244751</td>\n",
       "      <td>0.476562</td>\n",
       "      <td>0.201904</td>\n",
       "      <td>0.313232</td>\n",
       "      <td>0.724121</td>\n",
       "      <td>0.298340</td>\n",
       "      <td>...</td>\n",
       "      <td>0.475098</td>\n",
       "      <td>0.725586</td>\n",
       "      <td>0.126709</td>\n",
       "      <td>0.531250</td>\n",
       "      <td>0.348145</td>\n",
       "      <td>0.451416</td>\n",
       "      <td>0.611328</td>\n",
       "      <td>0.517578</td>\n",
       "      <td>0.191162</td>\n",
       "      <td>0.196289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000792</th>\n",
       "      <td>0.394531</td>\n",
       "      <td>0.664062</td>\n",
       "      <td>0.309082</td>\n",
       "      <td>0.400146</td>\n",
       "      <td>0.578125</td>\n",
       "      <td>0.287354</td>\n",
       "      <td>0.289307</td>\n",
       "      <td>0.165039</td>\n",
       "      <td>0.557617</td>\n",
       "      <td>0.630859</td>\n",
       "      <td>...</td>\n",
       "      <td>0.624512</td>\n",
       "      <td>0.830078</td>\n",
       "      <td>0.180908</td>\n",
       "      <td>0.480713</td>\n",
       "      <td>0.450928</td>\n",
       "      <td>0.481201</td>\n",
       "      <td>0.591309</td>\n",
       "      <td>0.458740</td>\n",
       "      <td>0.138428</td>\n",
       "      <td>0.197266</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3284 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0           BIOCARTA_GRANULOCYTES_PATHWAY  BIOCARTA_LYM_PATHWAY  \\\n",
       "Name                                                              \n",
       "ACH-001113                       0.201904              0.214722   \n",
       "ACH-000242                       0.210693              0.195068   \n",
       "ACH-000327                       0.226074              0.228394   \n",
       "ACH-000461                       0.230469              0.359375   \n",
       "ACH-000792                       0.394531              0.664062   \n",
       "\n",
       "0           BIOCARTA_BLYMPHOCYTE_PATHWAY  BIOCARTA_CARM_ER_PATHWAY  \\\n",
       "Name                                                                 \n",
       "ACH-001113                      0.107483                  0.436279   \n",
       "ACH-000242                      0.137573                  0.598633   \n",
       "ACH-000327                      0.131714                  0.807617   \n",
       "ACH-000461                      0.087646                  0.528809   \n",
       "ACH-000792                      0.309082                  0.400146   \n",
       "\n",
       "0           BIOCARTA_LAIR_PATHWAY  BIOCARTA_VDR_PATHWAY  \\\n",
       "Name                                                      \n",
       "ACH-001113               0.251221              0.543945   \n",
       "ACH-000242               0.192383              0.888672   \n",
       "ACH-000327               0.214966              0.105347   \n",
       "ACH-000461               0.244751              0.476562   \n",
       "ACH-000792               0.578125              0.287354   \n",
       "\n",
       "0           BIOCARTA_MTA3_PATHWAY  BIOCARTA_GABA_PATHWAY  \\\n",
       "Name                                                       \n",
       "ACH-001113               0.402344               0.446289   \n",
       "ACH-000242               0.381348               0.108154   \n",
       "ACH-000327               0.366455               0.534668   \n",
       "ACH-000461               0.201904               0.313232   \n",
       "ACH-000792               0.289307               0.165039   \n",
       "\n",
       "0           BIOCARTA_EGFR_SMRTE_PATHWAY  BIOCARTA_MONOCYTE_PATHWAY  ...  \\\n",
       "Name                                                                ...   \n",
       "ACH-001113                     0.692383                   0.299561  ...   \n",
       "ACH-000242                     0.679688                   0.285400  ...   \n",
       "ACH-000327                     0.508789                   0.279541  ...   \n",
       "ACH-000461                     0.724121                   0.298340  ...   \n",
       "ACH-000792                     0.557617                   0.630859  ...   \n",
       "\n",
       "0           HALLMARK_COAGULATION  HALLMARK_IL2_STAT5_SIGNALING  \\\n",
       "Name                                                             \n",
       "ACH-001113              0.369873                      0.589355   \n",
       "ACH-000242              0.448730                      0.651367   \n",
       "ACH-000327              0.377441                      0.626953   \n",
       "ACH-000461              0.475098                      0.725586   \n",
       "ACH-000792              0.624512                      0.830078   \n",
       "\n",
       "0           HALLMARK_BILE_ACID_METABOLISM  HALLMARK_PEROXISOME  \\\n",
       "Name                                                             \n",
       "ACH-001113                       0.231079             0.508301   \n",
       "ACH-000242                       0.461914             0.710938   \n",
       "ACH-000327                       0.639648             0.951172   \n",
       "ACH-000461                       0.126709             0.531250   \n",
       "ACH-000792                       0.180908             0.480713   \n",
       "\n",
       "0           HALLMARK_ALLOGRAFT_REJECTION  HALLMARK_SPERMATOGENESIS  \\\n",
       "Name                                                                 \n",
       "ACH-001113                      0.178223                  0.653809   \n",
       "ACH-000242                      0.337402                  0.380371   \n",
       "ACH-000327                      0.237305                  0.491943   \n",
       "ACH-000461                      0.348145                  0.451416   \n",
       "ACH-000792                      0.450928                  0.481201   \n",
       "\n",
       "0           HALLMARK_KRAS_SIGNALING  HALLMARK_KRAS_SIGNALING_UP  \\\n",
       "Name                                                              \n",
       "ACH-001113                 0.345215                    0.352539   \n",
       "ACH-000242                 0.503906                    0.462646   \n",
       "ACH-000327                 0.441162                    0.435791   \n",
       "ACH-000461                 0.611328                    0.517578   \n",
       "ACH-000792                 0.591309                    0.458740   \n",
       "\n",
       "0           HALLMARK_KRAS_SIGNALING_DN  HALLMARK_PANCREAS_BETA_CELLS  \n",
       "Name                                                                  \n",
       "ACH-001113                    0.390381                      0.233032  \n",
       "ACH-000242                    0.288818                      0.128052  \n",
       "ACH-000327                    0.353760                      0.339844  \n",
       "ACH-000461                    0.191162                      0.196289  \n",
       "ACH-000792                    0.138428                      0.197266  \n",
       "\n",
       "[5 rows x 3284 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter valid ID\n",
    "validSet = set()\n",
    "for name in trainData['Name']:\n",
    "    if CCLEidToCTRPNameDict[name] in cclToAUCdict:\n",
    "        validSet.add(name)\n",
    "trainData = trainData.loc[trainData['Name'].isin(validSet)]\n",
    "trainData = trainData.reset_index(drop = True)\n",
    "trainData = trainData.set_index(['Name'])\n",
    "trainData = trainData.apply(pd.to_numeric)\n",
    "trainData = normalize(trainData)\n",
    "trainData = trainData.astype('float16')\n",
    "trainData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c5f3e869",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 636 entries, ACH-001113 to ACH-000052\n",
      "Columns: 3284 entries, BIOCARTA_GRANULOCYTES_PATHWAY to HALLMARK_PANCREAS_BETA_CELLS\n",
      "dtypes: float16(3284)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "trainData.info(memory_usage = 'deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "90ce4112",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not all(trainData.columns == predictData.columns):\n",
    "    raise Exception(\"Column do not match!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "53f9ca95",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#labels data\n",
    "labelsDataOriginal = pd.DataFrame(columns = [sub[1] for sub in CTRPCCLAUC[7:]])\n",
    "for name in trainData.index:\n",
    "    labelsDataOriginal.loc[len(labelsDataOriginal.index)] = cclToAUCdict[CCLEidToCTRPNameDict[name]]\n",
    "labelsDataOriginal = labelsDataOriginal.set_index(trainData.index)\n",
    "labelsDataOriginal = labelsDataOriginal.astype('float16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "77d19250",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 636 entries, ACH-001113 to ACH-000052\n",
      "Columns: 481 entries, zebularine to GSK-J4\n",
      "dtypes: float16(481)\n",
      "memory usage: 639.1 KB\n"
     ]
    }
   ],
   "source": [
    "labelsDataOriginal.info(memory_usage = 'deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "651b49a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#add high priority at the begining. e.g dasatinib\n",
    "labels = list(labelsDataOriginal.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "062064dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#constants\n",
    "problem_types = ['regression'] \n",
    "eval_metrics = ['mean_squared_error']\n",
    "time_limit = 60 * 60 * 24\n",
    "tops = [100,300,600]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a029b871",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[WinError 3] 系统找不到指定的路径。: 'neuronal differentiation inducer III'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[18], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m label \u001b[38;5;129;01min\u001b[39;00m labels:\n\u001b[1;32m----> 2\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlabel\u001b[49m\u001b[43m)\u001b[49m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m8\u001b[39m:\n\u001b[0;32m      3\u001b[0m          \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[0;32m      5\u001b[0m     labelData \u001b[38;5;241m=\u001b[39m labelsDataOriginal[[label]]    \n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] 系统找不到指定的路径。: 'neuronal differentiation inducer III'"
     ]
    }
   ],
   "source": [
    "for label in labels:\n",
    "    if len(os.listdir(label)) > 8:\n",
    "         continue\n",
    "    try:\n",
    "        if len(os.listdir(label)) > 8:\n",
    "             continue\n",
    "    except:\n",
    "        print('Working on ' + label)\n",
    "    \n",
    "    labelData = labelsDataOriginal[[label]]    \n",
    "    trainDataSet = pd.concat([trainData, labelData], axis = 1)\n",
    "    \n",
    "    #first time training \n",
    "    save_path = label + '/' + 'GiloML_predictDrugAUC_Full_Feature_Medium_Quality_Model_' + label \n",
    "    multi_predictor = MultilabelPredictor(labels=[label], problem_types=problem_types, eval_metrics=eval_metrics, path=save_path)\n",
    "    \n",
    "    #Comment as it already trained\n",
    "    multi_predictor.fit(trainDataSet, time_limit=time_limit)\n",
    "    \n",
    "    multi_predictor = MultilabelPredictor.load(save_path)\n",
    "    result = multi_predictor.predict(predictData)\n",
    "    result.to_csv(label + '/' + 'GiloML_predictDrugAUC_Full_Feature_Medium_Quality_Result_' + label+ '.csv')\n",
    "    \n",
    "    #get feature importance\n",
    "    predictor = multi_predictor.get_predictor(label)\n",
    "    feature_importance = predictor.feature_importance(trainDataSet[trainDataSet[label] > float('-inf')], num_shuffle_sets = 3)\n",
    "    feature_importance.to_csv(label + '/' + \"GlioML_feature_importance_\" + label + \".csv\")\n",
    "    \n",
    "    #only consider feature has positive contribution\n",
    "    feature_importance = feature_importance[feature_importance['importance'] > 0]\n",
    "    \n",
    "    #Evaluation\n",
    "    evaaluateResult = []\n",
    "    for top in tops:\n",
    "        reducedfeatures = list(feature_importance[:top].index)\n",
    "        trainDataR =  trainData[reducedfeatures]\n",
    "        trainDataSetR = pd.concat([trainDataR,labelData],axis = 1)\n",
    "        save_path_R = label + '/' + 'GlioML_ReducedFeature_' + str(top) + '_' + label + '_FeatureEvaluationModel'\n",
    "        train, evaldata = train_test_split(trainDataSetR,test_size = 0.2,random_state=1112)\n",
    "        predictor_single = MultilabelPredictor(labels=[label], problem_types=problem_types, eval_metrics=eval_metrics, path=save_path_R)\n",
    "        predictor_single.fit(train, time_limit=time_limit)\n",
    "        # predictor_single = MultilabelPredictor.load(save_path_R)\n",
    "        evaluationData = predictor_single.evaluate(evaldata)\n",
    "        evaaluateResult.append(evaluationData[label]['mean_squared_error'])\n",
    "        if(top == 100):\n",
    "            save_path_final = label + '/' + 'GiloML_predictDrugAUC_Top_100_Features_Best_Quality_Full_Data_Model_' + label\n",
    "            predictor_single = MultilabelPredictor(labels = [label], problem_types = problem_types, eval_metrics = eval_metrics, path = save_path_final)\n",
    "            predictor_single.fit(trainDataSetR, final = True, time_limit = time_limit)\n",
    "            result_final = predictor_single.predict(predictData[reducedfeatures])\n",
    "            result_final.to_csv(save_path_final + \"_Result.csv\")\n",
    "    \n",
    "    maxPerformanceFeatureCount = tops[evaaluateResult.index(max(evaaluateResult))]\n",
    "    evaaluateResult = [['Component Name','TOP100','TOP300','TOP600'], [label] + evaaluateResult]\n",
    "    with open(label + '/' + 'GlioML_ReducedFeature_evalScore_' + label + '.csv', 'w', newline='') as csvfile:\n",
    "        writer = csv.writer(csvfile)\n",
    "        writer.writerows(evaaluateResult)\n",
    "    \n",
    "    if maxPerformanceFeatureCount > 100: \n",
    "        reducedfeatures = list(feature_importance[:maxPerformanceFeatureCount].index)\n",
    "        trainDataR =  trainData[reducedfeatures]\n",
    "        trainDataSetR = pd.concat([trainDataR,labelData],axis = 1)\n",
    "        save_path_final = label + '/' + 'GiloML_predictDrugAUC_Top_' + str(maxPerformanceFeatureCount) +'_Features_Best_Quality_Full_Data_Model_' + label\n",
    "        predictor_single = MultilabelPredictor(labels = [label], problem_types = problem_types, eval_metrics = eval_metrics, path = save_path_final)\n",
    "        predictor_single.fit(trainDataSetR, final = True, time_limit = time_limit)\n",
    "        result_final = predictor_single.predict(predictData[reducedfeatures])\n",
    "        result_final.to_csv(save_path_final + \"_Result.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48ffd795",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}