GlioML-MediumQuality.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa55bed2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:13:57.679151Z",
     "start_time": "2023-07-07T02:13:57.670150Z"
    }
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import collections\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from autogluon.tabular import TabularDataset, TabularPredictor\n",
    "from autogluon.common.utils.utils import setup_outputdir\n",
    "from autogluon.core.utils.loaders import load_pkl\n",
    "from autogluon.core.utils.savers import save_pkl\n",
    "import os.path\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from random import sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "41b72ee4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:13:57.710151Z",
     "start_time": "2023-07-07T02:13:57.681151Z"
    }
   },
   "outputs": [],
   "source": [
    "class MultilabelPredictor():\n",
    "    \"\"\" Tabular Predictor for predicting multiple columns in table.\n",
    "        Creates multiple TabularPredictor objects which you can also use individually.\n",
    "        You can access the TabularPredictor for a particular label via: `multilabel_predictor.get_predictor(label_i)`\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        labels : List[str]\n",
    "            The ith element of this list is the column (i.e. `label`) predicted by the ith TabularPredictor stored in this object.\n",
    "        path : str, default = None\n",
    "            Path to directory where models and intermediate outputs should be saved.\n",
    "            If unspecified, a time-stamped folder called \"AutogluonModels/ag-[TIMESTAMP]\" will be created in the working directory to store all models.\n",
    "            Note: To call `fit()` twice and save all results of each fit, you must specify different `path` locations or don't specify `path` at all.\n",
    "            Otherwise files from first `fit()` will be overwritten by second `fit()`.\n",
    "            Caution: when predicting many labels, this directory may grow large as it needs to store many TabularPredictors.\n",
    "        problem_types : List[str], default = None\n",
    "            The ith element is the `problem_type` for the ith TabularPredictor stored in this object.\n",
    "        eval_metrics : List[str], default = None\n",
    "            The ith element is the `eval_metric` for the ith TabularPredictor stored in this object.\n",
    "        consider_labels_correlation : bool, default = True\n",
    "            Whether the predictions of multiple labels should account for label correlations or predict each label independently of the others.\n",
    "            If True, the ordering of `labels` may affect resulting accuracy as each label is predicted conditional on the previous labels appearing earlier in this list (i.e. in an auto-regressive fashion).\n",
    "            Set to False if during inference you may want to individually use just the ith TabularPredictor without predicting all the other labels.\n",
    "        kwargs :\n",
    "            Arguments passed into the initialization of each TabularPredictor.\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    multi_predictor_file = 'multilabel_predictor.pkl'\n",
    "\n",
    "    def __init__(self, labels, path=None, problem_types=None, eval_metrics=None, consider_labels_correlation=False, **kwargs):\n",
    "        if (problem_types is not None) and (len(problem_types) != len(labels)):\n",
    "            raise ValueError(\"If provided, `problem_types` must have same length as `labels`\")\n",
    "        if (eval_metrics is not None) and (len(eval_metrics) != len(labels)):\n",
    "            raise ValueError(\"If provided, `eval_metrics` must have same length as `labels`\")\n",
    "        self.path = setup_outputdir(path, warn_if_exist=False)\n",
    "        self.labels = labels\n",
    "        self.consider_labels_correlation = consider_labels_correlation\n",
    "        self.predictors = {}  # key = label, value = TabularPredictor or str path to the TabularPredictor for this label\n",
    "        if eval_metrics is None:\n",
    "            self.eval_metrics = {}\n",
    "        else:\n",
    "            self.eval_metrics = {labels[i] : eval_metrics[i] for i in range(len(labels))}\n",
    "        problem_type = None\n",
    "        eval_metric = None\n",
    "        for i in range(len(labels)):\n",
    "            label = labels[i]\n",
    "            path_i = self.path + \"Predictor_\" + label\n",
    "            if problem_types is not None:\n",
    "                problem_type = problem_types[i]\n",
    "            if eval_metrics is not None:\n",
    "                eval_metric = eval_metrics[i]\n",
    "            self.predictors[label] = TabularPredictor(label=label, problem_type=problem_type, eval_metric=eval_metric, path=path_i, **kwargs)\n",
    "\n",
    "    def fit(self, train_data, tuning_data=None, final = False, **kwargs):\n",
    "        \"\"\" Fits a separate TabularPredictor to predict each of the labels.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            train_data, tuning_data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                See documentation for `TabularPredictor.fit()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `fit()` call for each TabularPredictor.\n",
    "        \"\"\"\n",
    "        if isinstance(train_data, str):\n",
    "            train_data = TabularDataset(train_data)\n",
    "        if tuning_data is not None and isinstance(tuning_data, str):\n",
    "            tuning_data = TabularDataset(tuning_data)\n",
    "        train_data_og = train_data.copy()\n",
    "        if tuning_data is not None:\n",
    "            tuning_data_og = tuning_data.copy()\n",
    "        else:\n",
    "            tuning_data_og = None\n",
    "        save_metrics = len(self.eval_metrics) == 0\n",
    "        start = time.time()\n",
    "        for i in range(len(self.labels)):\n",
    "            label = self.labels[i]\n",
    "            predictor = self.get_predictor(label)\n",
    "            if not self.consider_labels_correlation:\n",
    "                labels_to_drop = [l for l in self.labels if l != label]\n",
    "            else:\n",
    "                labels_to_drop = [self.labels[j] for j in range(i+1, len(self.labels))]\n",
    "            train_data = train_data_og.drop(labels_to_drop, axis=1)\n",
    "            if tuning_data is not None:\n",
    "                tuning_data = tuning_data_og.drop(labels_to_drop, axis=1)\n",
    "            print(f\"Fitting TabularPredictor for label: {label} ...{i / len(self.labels) * 100}%\")\n",
    "            print(f\"{(time.time() - start) / 60} minutes\")\n",
    "            if (final):\n",
    "                predictor.fit(train_data=train_data[train_data[label] > float('-inf')]\n",
    "                              , tuning_data = tuning_data\n",
    "                              ,presets = 'best_quality'\n",
    "                              ,num_bag_folds = 5,num_bag_sets = 2\n",
    "                              , **kwargs)\n",
    "            else:\n",
    "                predictor.fit(train_data=train_data[train_data[label] > float('-inf')]\n",
    "                              , tuning_data = tuning_data\n",
    "                              ,presets = 'medium_quality'\n",
    "                              #,presets = 'best_quality'\n",
    "                              #,num_bag_folds = 5,num_bag_sets = 2\n",
    "                              , **kwargs)\n",
    "            self.predictors[label] = predictor.path\n",
    "            if save_metrics:\n",
    "                self.eval_metrics[label] = predictor.eval_metric\n",
    "        self.save()\n",
    "\n",
    "    def predict(self, data, **kwargs):\n",
    "        \"\"\" Returns DataFrame with label columns containing predictions for each label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to make predictions for. If label columns are present in this data, they will be ignored. See documentation for `TabularPredictor.predict()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the predict() call for each TabularPredictor.\n",
    "        \"\"\"\n",
    "        return self._predict(data, as_proba=False, **kwargs)\n",
    "\n",
    "    def predict_proba(self, data, **kwargs):\n",
    "        \"\"\" Returns dict where each key is a label and the corresponding value is the `predict_proba()` output for just that label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to make predictions for. See documentation for `TabularPredictor.predict()` and `TabularPredictor.predict_proba()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `predict_proba()` call for each TabularPredictor (also passed into a `predict()` call).\n",
    "        \"\"\"\n",
    "        return self._predict(data, as_proba=True, **kwargs)\n",
    "\n",
    "    def evaluate(self, data, **kwargs):\n",
    "        \"\"\" Returns dict where each key is a label and the corresponding value is the `evaluate()` output for just that label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to evalate predictions of all labels for, must contain all labels as columns. See documentation for `TabularPredictor.evaluate()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `evaluate()` call for each TabularPredictor (also passed into the `predict()` call).\n",
    "        \"\"\"\n",
    "        data = self._get_data(data)\n",
    "        eval_dict = {}\n",
    "        for label in self.labels:\n",
    "            print(f\"Evaluating TabularPredictor for label: {label} ...\")\n",
    "            predictor = self.get_predictor(label)\n",
    "            \n",
    "            eval_dict[label] = predictor.evaluate(data[data[label] > float('-inf')], **kwargs)\n",
    "            if self.consider_labels_correlation:\n",
    "                data[label] = predictor.predict(data, **kwargs)\n",
    "        return eval_dict\n",
    "\n",
    "    def save(self):\n",
    "        \"\"\" Save MultilabelPredictor to disk. \"\"\"\n",
    "        for label in self.labels:\n",
    "            if not isinstance(self.predictors[label], str):\n",
    "                self.predictors[label] = self.predictors[label].path\n",
    "        save_pkl.save(path=self.path+self.multi_predictor_file, object=self)\n",
    "        print(f\"MultilabelPredictor saved to disk. Load with: MultilabelPredictor.load('{self.path}')\")\n",
    "\n",
    "    @classmethod\n",
    "    def load(cls, path):\n",
    "        \"\"\" Load MultilabelPredictor from disk `path` previously specified when creating this MultilabelPredictor. \"\"\"\n",
    "        path = os.path.expanduser(path)\n",
    "        if path[-1] != os.path.sep:\n",
    "            path = path + os.path.sep\n",
    "        return load_pkl.load(path=path+cls.multi_predictor_file)\n",
    "\n",
    "    def get_predictor(self, label):\n",
    "        \"\"\" Returns TabularPredictor which is used to predict this label. \"\"\"\n",
    "        predictor = self.predictors[label]\n",
    "        if isinstance(predictor, str):\n",
    "            return TabularPredictor.load(path=predictor)\n",
    "        return predictor\n",
    "\n",
    "    def _get_data(self, data):\n",
    "        if isinstance(data, str):\n",
    "            return TabularDataset(data)\n",
    "        return data.copy()\n",
    "\n",
    "    def _predict(self, data, as_proba=False, **kwargs):\n",
    "        data = self._get_data(data)\n",
    "        if as_proba:\n",
    "            predproba_dict = {}\n",
    "        for i,label in enumerate(self.labels):\n",
    "            print(f\"Predicting with TabularPredictor for label: {label} ...{i / len(self.labels) * 100}%\")\n",
    "            predictor = self.get_predictor(label)\n",
    "            if as_proba:\n",
    "                predproba_dict[label] = predictor.predict_proba(data, as_multiclass=True, **kwargs)\n",
    "            data[label] = predictor.predict(data, **kwargs)\n",
    "        if not as_proba:\n",
    "            return data[self.labels]\n",
    "        else:\n",
    "            return predproba_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d3ae94b9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:13:57.773154Z",
     "start_time": "2023-07-07T02:13:57.711152Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293\n",
      "187\n",
      "197\n",
      "1616\n",
      "665\n",
      "280\n",
      "53\n"
     ]
    }
   ],
   "source": [
    "#1 Load my ccl's ssGSEA signature\n",
    "myCCLSignature = []\n",
    "for name in ['sample.c2.cp.biocarta.gct',\n",
    "             'sample.c2.cp.kegg.gct',\n",
    "             'sample.c2.cp.pid.gct',\n",
    "             'sample.c2.cp.reactome.gct',\n",
    "             'sample.c2.cp.wiki.gct',\n",
    "             'sample.c6.gct',\n",
    "             'sample.hallmark.gct']:    \n",
    "    with open(name, mode ='r')as file:\n",
    "        csvFile = csv.reader(file)\n",
    "        CCLSignature = list(csvFile)[2:]\n",
    "        print(len(CCLSignature))\n",
    "    for i, row in enumerate(CCLSignature):\n",
    "        temp = CCLSignature[i][0].split('\\t')\n",
    "        if i > 0:\n",
    "            CCLSignature[i] = [temp[0]] + [float(d) for d in temp[2:]]\n",
    "        else:\n",
    "            CCLSignature[i] = [temp[0]] + temp[2:]\n",
    "    if not myCCLSignature:\n",
    "        myCCLSignature += CCLSignature\n",
    "    else:\n",
    "        myCCLSignature += CCLSignature[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a1ebef18",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:13:59.083491Z",
     "start_time": "2023-07-07T02:13:57.775154Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293\n",
      "187\n",
      "197\n",
      "1616\n",
      "665\n",
      "280\n",
      "53\n"
     ]
    }
   ],
   "source": [
    "#2 Load CCLE ssGSEA signature\n",
    "CCLECCLSignature = []\n",
    "for name in ['ccle.c2.cp.biocarta.gct',\n",
    "             'ccle.c2.cp.kegg.gct',\n",
    "             'ccle.c2.cp.pid.gct',\n",
    "             'ccle.c2.cp.reactome.gct',\n",
    "             'ccle.c2.cp.wiki.gct',\n",
    "             'ccle.c6.gct',\n",
    "             'ccle.hallmark.gct']:\n",
    "    with open(name, mode ='r')as file:\n",
    "        csvFile = csv.reader(file)\n",
    "        CCLSignature = list(csvFile)[2:]\n",
    "        print(len(CCLSignature))\n",
    "    for i, row in enumerate(CCLSignature):\n",
    "        temp = CCLSignature[i][0].split('\\t')\n",
    "        CCLSignature[i] = [temp[0]] + temp[2:]\n",
    "    if not CCLECCLSignature:\n",
    "        CCLECCLSignature += CCLSignature\n",
    "    else:\n",
    "        CCLECCLSignature += CCLSignature[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ce386d0b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:13:59.242568Z",
     "start_time": "2023-07-07T02:13:59.084491Z"
    }
   },
   "outputs": [],
   "source": [
    "#3 Load CTRP cclName to AUC map\n",
    "cclToAUCdict = collections.defaultdict(list)\n",
    "with open('CTRP_CCL_AUC.gct', mode ='r') as file:\n",
    "    csvFile = csv.reader(file)\n",
    "    CTRPCCLAUC = list(csvFile)\n",
    "    CTRPCCLAUC = [''.join(sub).split('\\t') for sub in CTRPCCLAUC]\n",
    "    cclNames = CTRPCCLAUC[3][4:]\n",
    "\n",
    "for i,cclName in enumerate(cclNames):\n",
    "    cclToAUCdict[cclName] = [float( '-inf' if sub[4+i] == 'NaN' else sub[4+i]) for sub in CTRPCCLAUC[7:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dfc80f9d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:13:59.306571Z",
     "start_time": "2023-07-07T02:13:59.244569Z"
    }
   },
   "outputs": [],
   "source": [
    "#4 Load ccleID to ctrpName map\n",
    "CCLEidToCTRPNameDict = collections.defaultdict(str)\n",
    "CCLEidToDiseaseName = collections.defaultdict(str)\n",
    "with open('sample_info.csv', mode ='r') as file:\n",
    "    csvFile = csv.reader(file)\n",
    "    mapInfos = list(csvFile)\n",
    "    for mapInfo in mapInfos[1:]:\n",
    "        CCLEidToCTRPNameDict[mapInfo[0]] = mapInfo[2]  \n",
    "        CCLEidToDiseaseName[mapInfo[0]] = mapInfo[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8996ddbc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:13:59.322571Z",
     "start_time": "2023-07-07T02:13:59.308571Z"
    }
   },
   "outputs": [],
   "source": [
    "def normalize(df):\n",
    "    result = df.copy()\n",
    "    for feature_name in df.columns:\n",
    "        max_value = df[feature_name].max()\n",
    "        min_value = df[feature_name].min()\n",
    "        result[feature_name] = (df[feature_name] - min_value) / (max_value - min_value)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cc53a03e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:14:01.535625Z",
     "start_time": "2023-07-07T02:13:59.324571Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIOCARTA_GRANULOCYTES_PATHWAY</th>\n",
       "      <th>BIOCARTA_LYM_PATHWAY</th>\n",
       "      <th>BIOCARTA_BLYMPHOCYTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_CARM_ER_PATHWAY</th>\n",
       "      <th>BIOCARTA_LAIR_PATHWAY</th>\n",
       "      <th>BIOCARTA_VDR_PATHWAY</th>\n",
       "      <th>BIOCARTA_MTA3_PATHWAY</th>\n",
       "      <th>BIOCARTA_GABA_PATHWAY</th>\n",
       "      <th>BIOCARTA_EGFR_SMRTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_MONOCYTE_PATHWAY</th>\n",
       "      <th>...</th>\n",
       "      <th>HALLMARK_COAGULATION</th>\n",
       "      <th>HALLMARK_IL2_STAT5_SIGNALING</th>\n",
       "      <th>HALLMARK_BILE_ACID_METABOLISM</th>\n",
       "      <th>HALLMARK_PEROXISOME</th>\n",
       "      <th>HALLMARK_ALLOGRAFT_REJECTION</th>\n",
       "      <th>HALLMARK_SPERMATOGENESIS</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_UP</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_DN</th>\n",
       "      <th>HALLMARK_PANCREAS_BETA_CELLS</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>THP1_d3_B.TPM</th>\n",
       "      <td>0.991211</td>\n",
       "      <td>0.985279</td>\n",
       "      <td>0.984154</td>\n",
       "      <td>0.876359</td>\n",
       "      <td>0.993570</td>\n",
       "      <td>0.595547</td>\n",
       "      <td>0.475054</td>\n",
       "      <td>0.492397</td>\n",
       "      <td>0.594030</td>\n",
       "      <td>0.986427</td>\n",
       "      <td>...</td>\n",
       "      <td>0.675939</td>\n",
       "      <td>0.809839</td>\n",
       "      <td>0.680037</td>\n",
       "      <td>0.774811</td>\n",
       "      <td>0.928606</td>\n",
       "      <td>0.793392</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.880986</td>\n",
       "      <td>0.337452</td>\n",
       "      <td>0.670398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2D_1.TPM</th>\n",
       "      <td>0.160241</td>\n",
       "      <td>0.161473</td>\n",
       "      <td>0.059402</td>\n",
       "      <td>0.379321</td>\n",
       "      <td>0.045364</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.089837</td>\n",
       "      <td>0.554072</td>\n",
       "      <td>0.363435</td>\n",
       "      <td>0.187854</td>\n",
       "      <td>...</td>\n",
       "      <td>0.025288</td>\n",
       "      <td>0.065203</td>\n",
       "      <td>0.221749</td>\n",
       "      <td>0.194506</td>\n",
       "      <td>0.019127</td>\n",
       "      <td>0.555587</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.037313</td>\n",
       "      <td>0.343351</td>\n",
       "      <td>0.248108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TRId7CWa.quant.TPM</th>\n",
       "      <td>0.438166</td>\n",
       "      <td>0.443539</td>\n",
       "      <td>0.540397</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.506852</td>\n",
       "      <td>0.304020</td>\n",
       "      <td>0.583516</td>\n",
       "      <td>0.429940</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700186</td>\n",
       "      <td>...</td>\n",
       "      <td>0.927243</td>\n",
       "      <td>0.736486</td>\n",
       "      <td>0.484722</td>\n",
       "      <td>0.130770</td>\n",
       "      <td>0.436573</td>\n",
       "      <td>0.053547</td>\n",
       "      <td>0.354081</td>\n",
       "      <td>0.771492</td>\n",
       "      <td>0.968649</td>\n",
       "      <td>0.720934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>H7.TPM</th>\n",
       "      <td>0.553921</td>\n",
       "      <td>0.562553</td>\n",
       "      <td>0.428064</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.724494</td>\n",
       "      <td>0.403809</td>\n",
       "      <td>0.852908</td>\n",
       "      <td>0.493875</td>\n",
       "      <td>0.511236</td>\n",
       "      <td>0.531963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.822794</td>\n",
       "      <td>0.816438</td>\n",
       "      <td>0.113695</td>\n",
       "      <td>0.248351</td>\n",
       "      <td>0.476935</td>\n",
       "      <td>0.189215</td>\n",
       "      <td>0.544488</td>\n",
       "      <td>0.707475</td>\n",
       "      <td>0.643843</td>\n",
       "      <td>0.258149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>THP1_d3_A.TPM</th>\n",
       "      <td>0.992471</td>\n",
       "      <td>0.992554</td>\n",
       "      <td>0.973644</td>\n",
       "      <td>0.795964</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>0.548292</td>\n",
       "      <td>0.615737</td>\n",
       "      <td>0.447267</td>\n",
       "      <td>0.474747</td>\n",
       "      <td>0.989311</td>\n",
       "      <td>...</td>\n",
       "      <td>0.691364</td>\n",
       "      <td>0.826434</td>\n",
       "      <td>0.710767</td>\n",
       "      <td>0.877729</td>\n",
       "      <td>0.928190</td>\n",
       "      <td>0.791363</td>\n",
       "      <td>0.947641</td>\n",
       "      <td>0.933168</td>\n",
       "      <td>0.476588</td>\n",
       "      <td>0.819935</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3284 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0                   BIOCARTA_GRANULOCYTES_PATHWAY  BIOCARTA_LYM_PATHWAY  \\\n",
       "Name                                                                      \n",
       "THP1_d3_B.TPM                            0.991211              0.985279   \n",
       "2D_1.TPM                                 0.160241              0.161473   \n",
       "TRId7CWa.quant.TPM                       0.438166              0.443539   \n",
       "H7.TPM                                   0.553921              0.562553   \n",
       "THP1_d3_A.TPM                            0.992471              0.992554   \n",
       "\n",
       "0                   BIOCARTA_BLYMPHOCYTE_PATHWAY  BIOCARTA_CARM_ER_PATHWAY  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                           0.984154                  0.876359   \n",
       "2D_1.TPM                                0.059402                  0.379321   \n",
       "TRId7CWa.quant.TPM                      0.540397                  0.000000   \n",
       "H7.TPM                                  0.428064                  1.000000   \n",
       "THP1_d3_A.TPM                           0.973644                  0.795964   \n",
       "\n",
       "0                   BIOCARTA_LAIR_PATHWAY  BIOCARTA_VDR_PATHWAY  \\\n",
       "Name                                                              \n",
       "THP1_d3_B.TPM                    0.993570              0.595547   \n",
       "2D_1.TPM                         0.045364              1.000000   \n",
       "TRId7CWa.quant.TPM               0.506852              0.304020   \n",
       "H7.TPM                           0.724494              0.403809   \n",
       "THP1_d3_A.TPM                    0.999802              0.548292   \n",
       "\n",
       "0                   BIOCARTA_MTA3_PATHWAY  BIOCARTA_GABA_PATHWAY  \\\n",
       "Name                                                               \n",
       "THP1_d3_B.TPM                    0.475054               0.492397   \n",
       "2D_1.TPM                         0.089837               0.554072   \n",
       "TRId7CWa.quant.TPM               0.583516               0.429940   \n",
       "H7.TPM                           0.852908               0.493875   \n",
       "THP1_d3_A.TPM                    0.615737               0.447267   \n",
       "\n",
       "0                   BIOCARTA_EGFR_SMRTE_PATHWAY  BIOCARTA_MONOCYTE_PATHWAY  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                          0.594030                   0.986427   \n",
       "2D_1.TPM                               0.363435                   0.187854   \n",
       "TRId7CWa.quant.TPM                     1.000000                   0.700186   \n",
       "H7.TPM                                 0.511236                   0.531963   \n",
       "THP1_d3_A.TPM                          0.474747                   0.989311   \n",
       "\n",
       "0                   ...  HALLMARK_COAGULATION  HALLMARK_IL2_STAT5_SIGNALING  \\\n",
       "Name                ...                                                       \n",
       "THP1_d3_B.TPM       ...              0.675939                      0.809839   \n",
       "2D_1.TPM            ...              0.025288                      0.065203   \n",
       "TRId7CWa.quant.TPM  ...              0.927243                      0.736486   \n",
       "H7.TPM              ...              0.822794                      0.816438   \n",
       "THP1_d3_A.TPM       ...              0.691364                      0.826434   \n",
       "\n",
       "0                   HALLMARK_BILE_ACID_METABOLISM  HALLMARK_PEROXISOME  \\\n",
       "Name                                                                     \n",
       "THP1_d3_B.TPM                            0.680037             0.774811   \n",
       "2D_1.TPM                                 0.221749             0.194506   \n",
       "TRId7CWa.quant.TPM                       0.484722             0.130770   \n",
       "H7.TPM                                   0.113695             0.248351   \n",
       "THP1_d3_A.TPM                            0.710767             0.877729   \n",
       "\n",
       "0                   HALLMARK_ALLOGRAFT_REJECTION  HALLMARK_SPERMATOGENESIS  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                           0.928606                  0.793392   \n",
       "2D_1.TPM                                0.019127                  0.555587   \n",
       "TRId7CWa.quant.TPM                      0.436573                  0.053547   \n",
       "H7.TPM                                  0.476935                  0.189215   \n",
       "THP1_d3_A.TPM                           0.928190                  0.791363   \n",
       "\n",
       "0                   HALLMARK_KRAS_SIGNALING  HALLMARK_KRAS_SIGNALING_UP  \\\n",
       "Name                                                                      \n",
       "THP1_d3_B.TPM                      1.000000                    0.880986   \n",
       "2D_1.TPM                           0.000000                    0.037313   \n",
       "TRId7CWa.quant.TPM                 0.354081                    0.771492   \n",
       "H7.TPM                             0.544488                    0.707475   \n",
       "THP1_d3_A.TPM                      0.947641                    0.933168   \n",
       "\n",
       "0                   HALLMARK_KRAS_SIGNALING_DN  HALLMARK_PANCREAS_BETA_CELLS  \n",
       "Name                                                                          \n",
       "THP1_d3_B.TPM                         0.337452                      0.670398  \n",
       "2D_1.TPM                              0.343351                      0.248108  \n",
       "TRId7CWa.quant.TPM                    0.968649                      0.720934  \n",
       "H7.TPM                                0.643843                      0.258149  \n",
       "THP1_d3_A.TPM                         0.476588                      0.819935  \n",
       "\n",
       "[5 rows x 3284 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prediction data \n",
    "predictData = pd.DataFrame(data = myCCLSignature).transpose()\n",
    "new_header = predictData.iloc[0] \n",
    "predictData = predictData[1:] \n",
    "predictData.columns = new_header \n",
    "predictData = predictData.apply(pd.to_numeric, errors='ignore')\n",
    "predictData = predictData.set_index(['Name'])\n",
    "predictData = normalize(predictData)\n",
    "predictData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c6935a1a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:14:01.821073Z",
     "start_time": "2023-07-07T02:14:01.535625Z"
    }
   },
   "outputs": [],
   "source": [
    "# Prepare train set\n",
    "trainData = pd.DataFrame(data = CCLECCLSignature).transpose()\n",
    "new_header = trainData.iloc[0] \n",
    "trainData = trainData[1:] \n",
    "trainData.columns = new_header "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "52014506",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:14:15.669565Z",
     "start_time": "2023-07-07T02:14:01.823073Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIOCARTA_GRANULOCYTES_PATHWAY</th>\n",
       "      <th>BIOCARTA_LYM_PATHWAY</th>\n",
       "      <th>BIOCARTA_BLYMPHOCYTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_CARM_ER_PATHWAY</th>\n",
       "      <th>BIOCARTA_LAIR_PATHWAY</th>\n",
       "      <th>BIOCARTA_VDR_PATHWAY</th>\n",
       "      <th>BIOCARTA_MTA3_PATHWAY</th>\n",
       "      <th>BIOCARTA_GABA_PATHWAY</th>\n",
       "      <th>BIOCARTA_EGFR_SMRTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_MONOCYTE_PATHWAY</th>\n",
       "      <th>...</th>\n",
       "      <th>HALLMARK_COAGULATION</th>\n",
       "      <th>HALLMARK_IL2_STAT5_SIGNALING</th>\n",
       "      <th>HALLMARK_BILE_ACID_METABOLISM</th>\n",
       "      <th>HALLMARK_PEROXISOME</th>\n",
       "      <th>HALLMARK_ALLOGRAFT_REJECTION</th>\n",
       "      <th>HALLMARK_SPERMATOGENESIS</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_UP</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_DN</th>\n",
       "      <th>HALLMARK_PANCREAS_BETA_CELLS</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ACH-001113</th>\n",
       "      <td>0.201899</td>\n",
       "      <td>0.214778</td>\n",
       "      <td>0.107506</td>\n",
       "      <td>0.436254</td>\n",
       "      <td>0.251323</td>\n",
       "      <td>0.543866</td>\n",
       "      <td>0.402230</td>\n",
       "      <td>0.446239</td>\n",
       "      <td>0.692287</td>\n",
       "      <td>0.299681</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369956</td>\n",
       "      <td>0.589361</td>\n",
       "      <td>0.231132</td>\n",
       "      <td>0.508302</td>\n",
       "      <td>0.178205</td>\n",
       "      <td>0.653705</td>\n",
       "      <td>0.345327</td>\n",
       "      <td>0.352541</td>\n",
       "      <td>0.390359</td>\n",
       "      <td>0.232997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000242</th>\n",
       "      <td>0.210679</td>\n",
       "      <td>0.195089</td>\n",
       "      <td>0.137566</td>\n",
       "      <td>0.598515</td>\n",
       "      <td>0.192437</td>\n",
       "      <td>0.888515</td>\n",
       "      <td>0.381405</td>\n",
       "      <td>0.108177</td>\n",
       "      <td>0.679700</td>\n",
       "      <td>0.285389</td>\n",
       "      <td>...</td>\n",
       "      <td>0.448744</td>\n",
       "      <td>0.651476</td>\n",
       "      <td>0.461799</td>\n",
       "      <td>0.710828</td>\n",
       "      <td>0.337337</td>\n",
       "      <td>0.380484</td>\n",
       "      <td>0.503869</td>\n",
       "      <td>0.462545</td>\n",
       "      <td>0.288830</td>\n",
       "      <td>0.128070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000327</th>\n",
       "      <td>0.226109</td>\n",
       "      <td>0.228415</td>\n",
       "      <td>0.131664</td>\n",
       "      <td>0.807736</td>\n",
       "      <td>0.214956</td>\n",
       "      <td>0.105343</td>\n",
       "      <td>0.366447</td>\n",
       "      <td>0.534534</td>\n",
       "      <td>0.508604</td>\n",
       "      <td>0.279435</td>\n",
       "      <td>...</td>\n",
       "      <td>0.377387</td>\n",
       "      <td>0.627130</td>\n",
       "      <td>0.639671</td>\n",
       "      <td>0.951057</td>\n",
       "      <td>0.237352</td>\n",
       "      <td>0.491974</td>\n",
       "      <td>0.441105</td>\n",
       "      <td>0.435839</td>\n",
       "      <td>0.353783</td>\n",
       "      <td>0.339955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000461</th>\n",
       "      <td>0.230496</td>\n",
       "      <td>0.359472</td>\n",
       "      <td>0.087643</td>\n",
       "      <td>0.529038</td>\n",
       "      <td>0.244749</td>\n",
       "      <td>0.476536</td>\n",
       "      <td>0.201961</td>\n",
       "      <td>0.313338</td>\n",
       "      <td>0.724363</td>\n",
       "      <td>0.298408</td>\n",
       "      <td>...</td>\n",
       "      <td>0.475189</td>\n",
       "      <td>0.725638</td>\n",
       "      <td>0.126653</td>\n",
       "      <td>0.531472</td>\n",
       "      <td>0.348210</td>\n",
       "      <td>0.451362</td>\n",
       "      <td>0.611400</td>\n",
       "      <td>0.517533</td>\n",
       "      <td>0.191123</td>\n",
       "      <td>0.196242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000792</th>\n",
       "      <td>0.394568</td>\n",
       "      <td>0.663835</td>\n",
       "      <td>0.308974</td>\n",
       "      <td>0.400182</td>\n",
       "      <td>0.578346</td>\n",
       "      <td>0.287388</td>\n",
       "      <td>0.289313</td>\n",
       "      <td>0.165095</td>\n",
       "      <td>0.557640</td>\n",
       "      <td>0.630891</td>\n",
       "      <td>...</td>\n",
       "      <td>0.624730</td>\n",
       "      <td>0.829904</td>\n",
       "      <td>0.180853</td>\n",
       "      <td>0.480622</td>\n",
       "      <td>0.450919</td>\n",
       "      <td>0.481082</td>\n",
       "      <td>0.591145</td>\n",
       "      <td>0.458782</td>\n",
       "      <td>0.138390</td>\n",
       "      <td>0.197292</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3284 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0           BIOCARTA_GRANULOCYTES_PATHWAY  BIOCARTA_LYM_PATHWAY  \\\n",
       "Name                                                              \n",
       "ACH-001113                       0.201899              0.214778   \n",
       "ACH-000242                       0.210679              0.195089   \n",
       "ACH-000327                       0.226109              0.228415   \n",
       "ACH-000461                       0.230496              0.359472   \n",
       "ACH-000792                       0.394568              0.663835   \n",
       "\n",
       "0           BIOCARTA_BLYMPHOCYTE_PATHWAY  BIOCARTA_CARM_ER_PATHWAY  \\\n",
       "Name                                                                 \n",
       "ACH-001113                      0.107506                  0.436254   \n",
       "ACH-000242                      0.137566                  0.598515   \n",
       "ACH-000327                      0.131664                  0.807736   \n",
       "ACH-000461                      0.087643                  0.529038   \n",
       "ACH-000792                      0.308974                  0.400182   \n",
       "\n",
       "0           BIOCARTA_LAIR_PATHWAY  BIOCARTA_VDR_PATHWAY  \\\n",
       "Name                                                      \n",
       "ACH-001113               0.251323              0.543866   \n",
       "ACH-000242               0.192437              0.888515   \n",
       "ACH-000327               0.214956              0.105343   \n",
       "ACH-000461               0.244749              0.476536   \n",
       "ACH-000792               0.578346              0.287388   \n",
       "\n",
       "0           BIOCARTA_MTA3_PATHWAY  BIOCARTA_GABA_PATHWAY  \\\n",
       "Name                                                       \n",
       "ACH-001113               0.402230               0.446239   \n",
       "ACH-000242               0.381405               0.108177   \n",
       "ACH-000327               0.366447               0.534534   \n",
       "ACH-000461               0.201961               0.313338   \n",
       "ACH-000792               0.289313               0.165095   \n",
       "\n",
       "0           BIOCARTA_EGFR_SMRTE_PATHWAY  BIOCARTA_MONOCYTE_PATHWAY  ...  \\\n",
       "Name                                                                ...   \n",
       "ACH-001113                     0.692287                   0.299681  ...   \n",
       "ACH-000242                     0.679700                   0.285389  ...   \n",
       "ACH-000327                     0.508604                   0.279435  ...   \n",
       "ACH-000461                     0.724363                   0.298408  ...   \n",
       "ACH-000792                     0.557640                   0.630891  ...   \n",
       "\n",
       "0           HALLMARK_COAGULATION  HALLMARK_IL2_STAT5_SIGNALING  \\\n",
       "Name                                                             \n",
       "ACH-001113              0.369956                      0.589361   \n",
       "ACH-000242              0.448744                      0.651476   \n",
       "ACH-000327              0.377387                      0.627130   \n",
       "ACH-000461              0.475189                      0.725638   \n",
       "ACH-000792              0.624730                      0.829904   \n",
       "\n",
       "0           HALLMARK_BILE_ACID_METABOLISM  HALLMARK_PEROXISOME  \\\n",
       "Name                                                             \n",
       "ACH-001113                       0.231132             0.508302   \n",
       "ACH-000242                       0.461799             0.710828   \n",
       "ACH-000327                       0.639671             0.951057   \n",
       "ACH-000461                       0.126653             0.531472   \n",
       "ACH-000792                       0.180853             0.480622   \n",
       "\n",
       "0           HALLMARK_ALLOGRAFT_REJECTION  HALLMARK_SPERMATOGENESIS  \\\n",
       "Name                                                                 \n",
       "ACH-001113                      0.178205                  0.653705   \n",
       "ACH-000242                      0.337337                  0.380484   \n",
       "ACH-000327                      0.237352                  0.491974   \n",
       "ACH-000461                      0.348210                  0.451362   \n",
       "ACH-000792                      0.450919                  0.481082   \n",
       "\n",
       "0           HALLMARK_KRAS_SIGNALING  HALLMARK_KRAS_SIGNALING_UP  \\\n",
       "Name                                                              \n",
       "ACH-001113                 0.345327                    0.352541   \n",
       "ACH-000242                 0.503869                    0.462545   \n",
       "ACH-000327                 0.441105                    0.435839   \n",
       "ACH-000461                 0.611400                    0.517533   \n",
       "ACH-000792                 0.591145                    0.458782   \n",
       "\n",
       "0           HALLMARK_KRAS_SIGNALING_DN  HALLMARK_PANCREAS_BETA_CELLS  \n",
       "Name                                                                  \n",
       "ACH-001113                    0.390359                      0.232997  \n",
       "ACH-000242                    0.288830                      0.128070  \n",
       "ACH-000327                    0.353783                      0.339955  \n",
       "ACH-000461                    0.191123                      0.196242  \n",
       "ACH-000792                    0.138390                      0.197292  \n",
       "\n",
       "[5 rows x 3284 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter valid ID\n",
    "validSet = set()\n",
    "for name in trainData['Name']:\n",
    "    if CCLEidToCTRPNameDict[name] in cclToAUCdict:\n",
    "        validSet.add(name)\n",
    "trainData = trainData.loc[trainData['Name'].isin(validSet)]\n",
    "trainData = trainData.reset_index(drop = True)\n",
    "trainData = trainData.set_index(['Name'])\n",
    "trainData = trainData.apply(pd.to_numeric)\n",
    "trainData = normalize(trainData)\n",
    "trainData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "90ce4112",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:14:15.685078Z",
     "start_time": "2023-07-07T02:14:15.670566Z"
    }
   },
   "outputs": [],
   "source": [
    "if not all(trainData.columns == predictData.columns):\n",
    "    raise Exception(\"Column do not match!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "53f9ca95",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:14:16.353257Z",
     "start_time": "2023-07-07T02:14:15.687078Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#labels data\n",
    "labelsDataOriginal = pd.DataFrame(columns = [sub[1] for sub in CTRPCCLAUC[7:]])\n",
    "for name in trainData.index:\n",
    "    labelsDataOriginal.loc[len(labelsDataOriginal.index)] = cclToAUCdict[CCLEidToCTRPNameDict[name]]\n",
    "labelsDataOriginal = labelsDataOriginal.set_index(trainData.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "277af282",
   "metadata": {},
   "outputs": [],
   "source": [
    "# labelsDataOriginal.rename(columns={\"neuronal differentiation inducer III\": \"neuronal_differentiation_inducer_III\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "651b49a8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:14:16.369094Z",
     "start_time": "2023-07-07T02:14:16.353257Z"
    }
   },
   "outputs": [],
   "source": [
    "#add high priority at the begining. e.g dasatinib\n",
    "labels = list(labelsDataOriginal.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a2f329e7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:14:16.385095Z",
     "start_time": "2023-07-07T02:14:16.371095Z"
    }
   },
   "outputs": [],
   "source": [
    "visited = set()\n",
    "for entry in os.scandir(os.getcwd()):\n",
    "    if entry.is_dir():\n",
    "        visited.add(entry.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a029b871",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-07T02:14:16.417097Z",
     "start_time": "2023-07-07T02:14:16.403096Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neuronal differentiation inducer III\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'problem_types' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[16], line 12\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[38;5;66;03m#first time training \u001b[39;00m\n\u001b[0;32m     11\u001b[0m save_path \u001b[38;5;241m=\u001b[39m label \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mGiloML_predictDrugAUC_Full_Feature_Medium_Quality_Model_\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m+\u001b[39m label \n\u001b[1;32m---> 12\u001b[0m multi_predictor \u001b[38;5;241m=\u001b[39m MultilabelPredictor(labels\u001b[38;5;241m=\u001b[39m[label], problem_types\u001b[38;5;241m=\u001b[39m\u001b[43mproblem_types\u001b[49m, eval_metrics\u001b[38;5;241m=\u001b[39meval_metrics, path\u001b[38;5;241m=\u001b[39msave_path)\n\u001b[0;32m     13\u001b[0m multi_predictor\u001b[38;5;241m.\u001b[39mfit(trainDataSet, time_limit\u001b[38;5;241m=\u001b[39mtime_limit)\n\u001b[0;32m     15\u001b[0m \u001b[38;5;66;03m#multi_predictor = MultilabelPredictor.load(save_path)\u001b[39;00m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'problem_types' is not defined"
     ]
    }
   ],
   "source": [
    "for label in labels:\n",
    "    if label in visited:\n",
    "        continue\n",
    "    visited.add(label)\n",
    "    print(label)\n",
    "    \n",
    "    labelData = labelsDataOriginal[[label]]    \n",
    "    trainDataSet = pd.concat([trainData, labelData], axis = 1)\n",
    "    \n",
    "    #first time training \n",
    "    save_path = label + '/' + 'GiloML_predictDrugAUC_Full_Feature_Medium_Quality_Model_' + label \n",
    "    multi_predictor = MultilabelPredictor(labels=[label], problem_types=problem_types, eval_metrics=eval_metrics, path=save_path)\n",
    "    multi_predictor.fit(trainDataSet, time_limit=time_limit)\n",
    "    \n",
    "    #multi_predictor = MultilabelPredictor.load(save_path)\n",
    "    result = multi_predictor.predict(predictData)\n",
    "    result.to_csv(label + '/' + 'GiloML_predictDrugAUC_Full_Feature_Medium_Quality_Result_' + label+ '.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59f46b63",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}