GlioML-Main-EnhancedFeature_mutaionCountsMatter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa55bed2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import collections\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from autogluon.tabular import TabularDataset, TabularPredictor\n",
    "from autogluon.common.utils.utils import setup_outputdir\n",
    "from autogluon.core.utils.loaders import load_pkl\n",
    "from autogluon.core.utils.savers import save_pkl\n",
    "import os.path\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from random import sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "41b72ee4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultilabelPredictor():\n",
    "    \"\"\" Tabular Predictor for predicting multiple columns in table.\n",
    "        Creates multiple TabularPredictor objects which you can also use individually.\n",
    "        You can access the TabularPredictor for a particular label via: `multilabel_predictor.get_predictor(label_i)`\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        labels : List[str]\n",
    "            The ith element of this list is the column (i.e. `label`) predicted by the ith TabularPredictor stored in this object.\n",
    "        path : str, default = None\n",
    "            Path to directory where models and intermediate outputs should be saved.\n",
    "            If unspecified, a time-stamped folder called \"AutogluonModels/ag-[TIMESTAMP]\" will be created in the working directory to store all models.\n",
    "            Note: To call `fit()` twice and save all results of each fit, you must specify different `path` locations or don't specify `path` at all.\n",
    "            Otherwise files from first `fit()` will be overwritten by second `fit()`.\n",
    "            Caution: when predicting many labels, this directory may grow large as it needs to store many TabularPredictors.\n",
    "        problem_types : List[str], default = None\n",
    "            The ith element is the `problem_type` for the ith TabularPredictor stored in this object.\n",
    "        eval_metrics : List[str], default = None\n",
    "            The ith element is the `eval_metric` for the ith TabularPredictor stored in this object.\n",
    "        consider_labels_correlation : bool, default = True\n",
    "            Whether the predictions of multiple labels should account for label correlations or predict each label independently of the others.\n",
    "            If True, the ordering of `labels` may affect resulting accuracy as each label is predicted conditional on the previous labels appearing earlier in this list (i.e. in an auto-regressive fashion).\n",
    "            Set to False if during inference you may want to individually use just the ith TabularPredictor without predicting all the other labels.\n",
    "        kwargs :\n",
    "            Arguments passed into the initialization of each TabularPredictor.\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    multi_predictor_file = 'multilabel_predictor.pkl'\n",
    "\n",
    "    def __init__(self, labels, path=None, problem_types=None, eval_metrics=None, consider_labels_correlation=False, **kwargs):\n",
    "        if (problem_types is not None) and (len(problem_types) != len(labels)):\n",
    "            raise ValueError(\"If provided, `problem_types` must have same length as `labels`\")\n",
    "        if (eval_metrics is not None) and (len(eval_metrics) != len(labels)):\n",
    "            raise ValueError(\"If provided, `eval_metrics` must have same length as `labels`\")\n",
    "        self.path = setup_outputdir(path, warn_if_exist=False)\n",
    "        self.labels = labels\n",
    "        self.consider_labels_correlation = consider_labels_correlation\n",
    "        self.predictors = {}  # key = label, value = TabularPredictor or str path to the TabularPredictor for this label\n",
    "        if eval_metrics is None:\n",
    "            self.eval_metrics = {}\n",
    "        else:\n",
    "            self.eval_metrics = {labels[i] : eval_metrics[i] for i in range(len(labels))}\n",
    "        problem_type = None\n",
    "        eval_metric = None\n",
    "        for i in range(len(labels)):\n",
    "            label = labels[i]\n",
    "            path_i = self.path + \"Predictor_\" + label\n",
    "            if problem_types is not None:\n",
    "                problem_type = problem_types[i]\n",
    "            if eval_metrics is not None:\n",
    "                eval_metric = eval_metrics[i]\n",
    "            self.predictors[label] = TabularPredictor(label=label, problem_type=problem_type, eval_metric=eval_metric, path=path_i, **kwargs)\n",
    "\n",
    "    def fit(self, train_data, tuning_data=None, final = False, **kwargs):\n",
    "        \"\"\" Fits a separate TabularPredictor to predict each of the labels.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            train_data, tuning_data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                See documentation for `TabularPredictor.fit()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `fit()` call for each TabularPredictor.\n",
    "        \"\"\"\n",
    "        if isinstance(train_data, str):\n",
    "            train_data = TabularDataset(train_data)\n",
    "        if tuning_data is not None and isinstance(tuning_data, str):\n",
    "            tuning_data = TabularDataset(tuning_data)\n",
    "        train_data_og = train_data.copy()\n",
    "        if tuning_data is not None:\n",
    "            tuning_data_og = tuning_data.copy()\n",
    "        else:\n",
    "            tuning_data_og = None\n",
    "        save_metrics = len(self.eval_metrics) == 0\n",
    "        start = time.time()\n",
    "        for i in range(len(self.labels)):\n",
    "            label = self.labels[i]\n",
    "            predictor = self.get_predictor(label)\n",
    "            if not self.consider_labels_correlation:\n",
    "                labels_to_drop = [l for l in self.labels if l != label]\n",
    "            else:\n",
    "                labels_to_drop = [self.labels[j] for j in range(i+1, len(self.labels))]\n",
    "            train_data = train_data_og.drop(labels_to_drop, axis=1)\n",
    "            if tuning_data is not None:\n",
    "                tuning_data = tuning_data_og.drop(labels_to_drop, axis=1)\n",
    "            print(f\"Fitting TabularPredictor for label: {label} ...{i / len(self.labels) * 100}%\")\n",
    "            print(f\"{(time.time() - start) / 60} minutes\")\n",
    "            if (final):\n",
    "                predictor.fit(train_data=train_data[train_data[label] > float('-inf')]\n",
    "                              , tuning_data = tuning_data\n",
    "                              ,presets = 'best_quality'\n",
    "                              ,num_bag_folds = 5,num_bag_sets = 2\n",
    "                              , **kwargs)\n",
    "            else:\n",
    "                predictor.fit(train_data=train_data[train_data[label] > float('-inf')]\n",
    "                              , tuning_data = tuning_data\n",
    "                              ,presets = 'medium_quality'\n",
    "                              #,presets = 'best_quality'\n",
    "                              #,num_bag_folds = 5,num_bag_sets = 2\n",
    "                              , **kwargs)\n",
    "            self.predictors[label] = predictor.path\n",
    "            if save_metrics:\n",
    "                self.eval_metrics[label] = predictor.eval_metric\n",
    "        self.save()\n",
    "\n",
    "    def predict(self, data, **kwargs):\n",
    "        \"\"\" Returns DataFrame with label columns containing predictions for each label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to make predictions for. If label columns are present in this data, they will be ignored. See documentation for `TabularPredictor.predict()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the predict() call for each TabularPredictor.\n",
    "        \"\"\"\n",
    "        return self._predict(data, as_proba=False, **kwargs)\n",
    "\n",
    "    def predict_proba(self, data, **kwargs):\n",
    "        \"\"\" Returns dict where each key is a label and the corresponding value is the `predict_proba()` output for just that label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to make predictions for. See documentation for `TabularPredictor.predict()` and `TabularPredictor.predict_proba()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `predict_proba()` call for each TabularPredictor (also passed into a `predict()` call).\n",
    "        \"\"\"\n",
    "        return self._predict(data, as_proba=True, **kwargs)\n",
    "\n",
    "    def evaluate(self, data, **kwargs):\n",
    "        \"\"\" Returns dict where each key is a label and the corresponding value is the `evaluate()` output for just that label.\n",
    "\n",
    "            Parameters\n",
    "            ----------\n",
    "            data : str or autogluon.tabular.TabularDataset or pd.DataFrame\n",
    "                Data to evalate predictions of all labels for, must contain all labels as columns. See documentation for `TabularPredictor.evaluate()`.\n",
    "            kwargs :\n",
    "                Arguments passed into the `evaluate()` call for each TabularPredictor (also passed into the `predict()` call).\n",
    "        \"\"\"\n",
    "        data = self._get_data(data)\n",
    "        eval_dict = {}\n",
    "        for label in self.labels:\n",
    "            print(f\"Evaluating TabularPredictor for label: {label} ...\")\n",
    "            predictor = self.get_predictor(label)\n",
    "            \n",
    "            eval_dict[label] = predictor.evaluate(data[data[label] > float('-inf')], **kwargs)\n",
    "            if self.consider_labels_correlation:\n",
    "                data[label] = predictor.predict(data, **kwargs)\n",
    "        return eval_dict\n",
    "\n",
    "    def save(self):\n",
    "        \"\"\" Save MultilabelPredictor to disk. \"\"\"\n",
    "        for label in self.labels:\n",
    "            if not isinstance(self.predictors[label], str):\n",
    "                self.predictors[label] = self.predictors[label].path\n",
    "        save_pkl.save(path=self.path+self.multi_predictor_file, object=self)\n",
    "        print(f\"MultilabelPredictor saved to disk. Load with: MultilabelPredictor.load('{self.path}')\")\n",
    "\n",
    "    @classmethod\n",
    "    def load(cls, path):\n",
    "        \"\"\" Load MultilabelPredictor from disk `path` previously specified when creating this MultilabelPredictor. \"\"\"\n",
    "        path = os.path.expanduser(path)\n",
    "        if path[-1] != os.path.sep:\n",
    "            path = path + os.path.sep\n",
    "        return load_pkl.load(path=path+cls.multi_predictor_file)\n",
    "\n",
    "    def get_predictor(self, label):\n",
    "        \"\"\" Returns TabularPredictor which is used to predict this label. \"\"\"\n",
    "        predictor = self.predictors[label]\n",
    "        if isinstance(predictor, str):\n",
    "            return TabularPredictor.load(path=predictor)\n",
    "        return predictor\n",
    "\n",
    "    def _get_data(self, data):\n",
    "        if isinstance(data, str):\n",
    "            return TabularDataset(data)\n",
    "        return data.copy()\n",
    "\n",
    "    def _predict(self, data, as_proba=False, **kwargs):\n",
    "        data = self._get_data(data)\n",
    "        if as_proba:\n",
    "            predproba_dict = {}\n",
    "        for i,label in enumerate(self.labels):\n",
    "            print(f\"Predicting with TabularPredictor for label: {label} ...{i / len(self.labels) * 100}%\")\n",
    "            predictor = self.get_predictor(label)\n",
    "            if as_proba:\n",
    "                predproba_dict[label] = predictor.predict_proba(data, as_multiclass=True, **kwargs)\n",
    "            data[label] = predictor.predict(data, **kwargs)\n",
    "        if not as_proba:\n",
    "            return data[self.labels]\n",
    "        else:\n",
    "            return predproba_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d3ae94b9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-02T06:09:54.216156Z",
     "start_time": "2022-07-02T06:09:54.210855Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293\n",
      "187\n",
      "197\n",
      "1616\n",
      "665\n",
      "280\n",
      "53\n"
     ]
    }
   ],
   "source": [
    "#1 Load my ccl's ssGSEA signature\n",
    "myCCLSignature = []\n",
    "for name in ['sample.c2.cp.biocarta.gct',\n",
    "             'sample.c2.cp.kegg.gct',\n",
    "             'sample.c2.cp.pid.gct',\n",
    "             'sample.c2.cp.reactome.gct',\n",
    "             'sample.c2.cp.wiki.gct',\n",
    "             'sample.c6.gct',\n",
    "             'sample.hallmark.gct']:    \n",
    "    with open(name, mode ='r')as file:\n",
    "        csvFile = csv.reader(file)\n",
    "        CCLSignature = list(csvFile)[2:]\n",
    "        print(len(CCLSignature))\n",
    "    for i, row in enumerate(CCLSignature):\n",
    "        temp = CCLSignature[i][0].split('\\t')\n",
    "        if i > 0:\n",
    "            CCLSignature[i] = [temp[0]] + [float(d) for d in temp[2:]]\n",
    "        else:\n",
    "            CCLSignature[i] = [temp[0]] + temp[2:]\n",
    "    if not myCCLSignature:\n",
    "        myCCLSignature += CCLSignature\n",
    "    else:\n",
    "        myCCLSignature += CCLSignature[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a1ebef18",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-02T06:10:01.747913Z",
     "start_time": "2022-07-02T06:10:01.683560Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293\n",
      "187\n",
      "197\n",
      "1616\n",
      "665\n",
      "280\n",
      "53\n"
     ]
    }
   ],
   "source": [
    "#2 Load CCLE ssGSEA signature\n",
    "CCLECCLSignature = []\n",
    "for name in ['ccle.c2.cp.biocarta.gct',\n",
    "             'ccle.c2.cp.kegg.gct',\n",
    "             'ccle.c2.cp.pid.gct',\n",
    "             'ccle.c2.cp.reactome.gct',\n",
    "             'ccle.c2.cp.wiki.gct',\n",
    "             'ccle.c6.gct',\n",
    "             'ccle.hallmark.gct']:\n",
    "    with open(name, mode ='r')as file:\n",
    "        csvFile = csv.reader(file)\n",
    "        CCLSignature = list(csvFile)[2:]\n",
    "        print(len(CCLSignature))\n",
    "    for i, row in enumerate(CCLSignature):\n",
    "        temp = CCLSignature[i][0].split('\\t')\n",
    "        CCLSignature[i] = [temp[0]] + temp[2:]\n",
    "    if not CCLECCLSignature:\n",
    "        CCLECCLSignature += CCLSignature\n",
    "    else:\n",
    "        CCLECCLSignature += CCLSignature[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0765f45a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "74b85673",
   "metadata": {},
   "outputs": [],
   "source": [
    "############new "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e6136f3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#2.1 load model\n",
    "modelMetaData = pd.read_csv('Model.csv', header=0)\n",
    "modelMetaData = modelMetaData[['ModelID','Age','Sex']]\n",
    "sex_mapping = {'Male': 1, 'Female': 0}\n",
    "modelMetaData['Sex'] = modelMetaData['Sex'].map(sex_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "07fb2549",
   "metadata": {},
   "outputs": [],
   "source": [
    "metaDataDict = {}\n",
    "for index, row in modelMetaData.iterrows():\n",
    "    key = row['ModelID'] \n",
    "    value = [row['Age'], row['Sex']] \n",
    "    if key not in metaDataDict:\n",
    "        metaDataDict[key] = value\n",
    "    else:\n",
    "        raise Exception(\"duplicated Id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2b62ef3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/y4/nb52fxtx6g934cctxtd11mz40000gn/T/ipykernel_3926/2322606452.py:2: DtypeWarning: Columns (49,50,51,53) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  mutationData = pd.read_csv('OmicsSomaticMutations.csv')\n"
     ]
    }
   ],
   "source": [
    "#2.2 load Mutation\n",
    "mutationData = pd.read_csv('OmicsSomaticMutations.csv')\n",
    "mutationData = mutationData[['ModelID','HugoSymbol']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b80e741d",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidateKeyList = [\n",
    "    'MGMT', 'IDH1', 'IDH2', 'EGFR',\n",
    "    'TTN', 'MAPRE3', 'TP53', 'PIK3C2B', 'CIC', 'LRP2', 'LRP1', 'NRXN2', 'TEAD2', 'MYH3', 'NOTCH1', 'TFE3', 'PIK3R1', 'FRMD4A', 'PRCC', 'CHD3', 'BAG6', 'GLYR1', 'ADAM23', 'MSH6', 'ATRX',\n",
    "    'MUC16', 'PTEN', 'NF1', 'OBSN', 'FLG', 'RYR2', 'MUC17',\n",
    "    'BRAF', 'CDKN2A', 'CDKN2B', 'TERT', 'MYC'\n",
    "]\n",
    "candidateGeneMutationCount = {key: 0 for key in candidateKeyList}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a560b348",
   "metadata": {},
   "outputs": [],
   "source": [
    "mutationDataDict = {}\n",
    "for index, row in mutationData.iterrows():\n",
    "    key = row['ModelID']\n",
    "    if key not in mutationDataDict:\n",
    "        mutationDataDict[key] = candidateGeneMutationCount.copy()\n",
    "    gene = row['HugoSymbol']\n",
    "    if gene in candidateGeneMutationCount:\n",
    "        mutationDataDict[key][row['HugoSymbol']] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cc7b0a4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#############new end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ce386d0b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-02T06:10:47.896499Z",
     "start_time": "2022-07-02T06:10:47.530996Z"
    }
   },
   "outputs": [],
   "source": [
    "#3 Load CTRP cclName to AUC map\n",
    "cclToAUCdict = collections.defaultdict(list)\n",
    "with open('CTRP_CCL_AUC.gct', mode ='r') as file:\n",
    "    csvFile = csv.reader(file)\n",
    "    CTRPCCLAUC = list(csvFile)\n",
    "    CTRPCCLAUC = [''.join(sub).split('\\t') for sub in CTRPCCLAUC]\n",
    "    cclNames = CTRPCCLAUC[3][4:]\n",
    "\n",
    "for i,cclName in enumerate(cclNames):\n",
    "    cclToAUCdict[cclName] = [float( '-inf' if sub[4+i] == 'NaN' else sub[4+i]) for sub in CTRPCCLAUC[7:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dfc80f9d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-02T06:10:57.926109Z",
     "start_time": "2022-07-02T06:10:57.915764Z"
    }
   },
   "outputs": [],
   "source": [
    "#4 Load ccleID to ctrpName map\n",
    "CCLEidToCTRPNameDict = collections.defaultdict(str)\n",
    "CCLEidToDiseaseName = collections.defaultdict(str)\n",
    "with open('sample_info.csv', mode ='r') as file:\n",
    "    csvFile = csv.reader(file)\n",
    "    mapInfos = list(csvFile)\n",
    "    for mapInfo in mapInfos[1:]:\n",
    "        CCLEidToCTRPNameDict[mapInfo[0]] = mapInfo[2]  \n",
    "        CCLEidToDiseaseName[mapInfo[0]] = mapInfo[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8996ddbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(df):\n",
    "    result = df.copy()\n",
    "    for feature_name in df.columns:\n",
    "        max_value = df[feature_name].max()\n",
    "        min_value = df[feature_name].min()\n",
    "        result[feature_name] = (df[feature_name] - min_value) / (max_value - min_value)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cc53a03e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIOCARTA_GRANULOCYTES_PATHWAY</th>\n",
       "      <th>BIOCARTA_LYM_PATHWAY</th>\n",
       "      <th>BIOCARTA_BLYMPHOCYTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_CARM_ER_PATHWAY</th>\n",
       "      <th>BIOCARTA_LAIR_PATHWAY</th>\n",
       "      <th>BIOCARTA_VDR_PATHWAY</th>\n",
       "      <th>BIOCARTA_MTA3_PATHWAY</th>\n",
       "      <th>BIOCARTA_GABA_PATHWAY</th>\n",
       "      <th>BIOCARTA_EGFR_SMRTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_MONOCYTE_PATHWAY</th>\n",
       "      <th>...</th>\n",
       "      <th>HALLMARK_COAGULATION</th>\n",
       "      <th>HALLMARK_IL2_STAT5_SIGNALING</th>\n",
       "      <th>HALLMARK_BILE_ACID_METABOLISM</th>\n",
       "      <th>HALLMARK_PEROXISOME</th>\n",
       "      <th>HALLMARK_ALLOGRAFT_REJECTION</th>\n",
       "      <th>HALLMARK_SPERMATOGENESIS</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_UP</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_DN</th>\n",
       "      <th>HALLMARK_PANCREAS_BETA_CELLS</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>THP1_d3_B.TPM</th>\n",
       "      <td>0.991211</td>\n",
       "      <td>0.985352</td>\n",
       "      <td>0.984375</td>\n",
       "      <td>0.876465</td>\n",
       "      <td>0.993652</td>\n",
       "      <td>0.595703</td>\n",
       "      <td>0.475098</td>\n",
       "      <td>0.492432</td>\n",
       "      <td>0.594238</td>\n",
       "      <td>0.986328</td>\n",
       "      <td>...</td>\n",
       "      <td>0.675781</td>\n",
       "      <td>0.810059</td>\n",
       "      <td>0.680176</td>\n",
       "      <td>0.774902</td>\n",
       "      <td>0.928711</td>\n",
       "      <td>0.793457</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.880859</td>\n",
       "      <td>0.337402</td>\n",
       "      <td>0.670410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2D_1.TPM</th>\n",
       "      <td>0.160278</td>\n",
       "      <td>0.161499</td>\n",
       "      <td>0.059387</td>\n",
       "      <td>0.379395</td>\n",
       "      <td>0.045349</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.089844</td>\n",
       "      <td>0.554199</td>\n",
       "      <td>0.363525</td>\n",
       "      <td>0.187866</td>\n",
       "      <td>...</td>\n",
       "      <td>0.025284</td>\n",
       "      <td>0.065186</td>\n",
       "      <td>0.221802</td>\n",
       "      <td>0.194458</td>\n",
       "      <td>0.019119</td>\n",
       "      <td>0.555664</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.037323</td>\n",
       "      <td>0.343262</td>\n",
       "      <td>0.248169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TRId7CWa.quant.TPM</th>\n",
       "      <td>0.438232</td>\n",
       "      <td>0.443604</td>\n",
       "      <td>0.540527</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.506836</td>\n",
       "      <td>0.303955</td>\n",
       "      <td>0.583496</td>\n",
       "      <td>0.429932</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700195</td>\n",
       "      <td>...</td>\n",
       "      <td>0.927246</td>\n",
       "      <td>0.736328</td>\n",
       "      <td>0.484619</td>\n",
       "      <td>0.130737</td>\n",
       "      <td>0.436523</td>\n",
       "      <td>0.053558</td>\n",
       "      <td>0.354004</td>\n",
       "      <td>0.771484</td>\n",
       "      <td>0.968750</td>\n",
       "      <td>0.720703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>H7.TPM</th>\n",
       "      <td>0.553711</td>\n",
       "      <td>0.562500</td>\n",
       "      <td>0.427979</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.724609</td>\n",
       "      <td>0.403809</td>\n",
       "      <td>0.853027</td>\n",
       "      <td>0.493896</td>\n",
       "      <td>0.511230</td>\n",
       "      <td>0.531738</td>\n",
       "      <td>...</td>\n",
       "      <td>0.822754</td>\n",
       "      <td>0.816406</td>\n",
       "      <td>0.113708</td>\n",
       "      <td>0.248291</td>\n",
       "      <td>0.477051</td>\n",
       "      <td>0.189209</td>\n",
       "      <td>0.544434</td>\n",
       "      <td>0.707520</td>\n",
       "      <td>0.644043</td>\n",
       "      <td>0.258057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>THP1_d3_A.TPM</th>\n",
       "      <td>0.992676</td>\n",
       "      <td>0.992676</td>\n",
       "      <td>0.973633</td>\n",
       "      <td>0.795898</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.548340</td>\n",
       "      <td>0.615723</td>\n",
       "      <td>0.447266</td>\n",
       "      <td>0.474854</td>\n",
       "      <td>0.989258</td>\n",
       "      <td>...</td>\n",
       "      <td>0.691406</td>\n",
       "      <td>0.826660</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>0.877930</td>\n",
       "      <td>0.928223</td>\n",
       "      <td>0.791504</td>\n",
       "      <td>0.947754</td>\n",
       "      <td>0.933105</td>\n",
       "      <td>0.476562</td>\n",
       "      <td>0.819824</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3284 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0                   BIOCARTA_GRANULOCYTES_PATHWAY  BIOCARTA_LYM_PATHWAY  \\\n",
       "Name                                                                      \n",
       "THP1_d3_B.TPM                            0.991211              0.985352   \n",
       "2D_1.TPM                                 0.160278              0.161499   \n",
       "TRId7CWa.quant.TPM                       0.438232              0.443604   \n",
       "H7.TPM                                   0.553711              0.562500   \n",
       "THP1_d3_A.TPM                            0.992676              0.992676   \n",
       "\n",
       "0                   BIOCARTA_BLYMPHOCYTE_PATHWAY  BIOCARTA_CARM_ER_PATHWAY  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                           0.984375                  0.876465   \n",
       "2D_1.TPM                                0.059387                  0.379395   \n",
       "TRId7CWa.quant.TPM                      0.540527                  0.000000   \n",
       "H7.TPM                                  0.427979                  1.000000   \n",
       "THP1_d3_A.TPM                           0.973633                  0.795898   \n",
       "\n",
       "0                   BIOCARTA_LAIR_PATHWAY  BIOCARTA_VDR_PATHWAY  \\\n",
       "Name                                                              \n",
       "THP1_d3_B.TPM                    0.993652              0.595703   \n",
       "2D_1.TPM                         0.045349              1.000000   \n",
       "TRId7CWa.quant.TPM               0.506836              0.303955   \n",
       "H7.TPM                           0.724609              0.403809   \n",
       "THP1_d3_A.TPM                    1.000000              0.548340   \n",
       "\n",
       "0                   BIOCARTA_MTA3_PATHWAY  BIOCARTA_GABA_PATHWAY  \\\n",
       "Name                                                               \n",
       "THP1_d3_B.TPM                    0.475098               0.492432   \n",
       "2D_1.TPM                         0.089844               0.554199   \n",
       "TRId7CWa.quant.TPM               0.583496               0.429932   \n",
       "H7.TPM                           0.853027               0.493896   \n",
       "THP1_d3_A.TPM                    0.615723               0.447266   \n",
       "\n",
       "0                   BIOCARTA_EGFR_SMRTE_PATHWAY  BIOCARTA_MONOCYTE_PATHWAY  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                          0.594238                   0.986328   \n",
       "2D_1.TPM                               0.363525                   0.187866   \n",
       "TRId7CWa.quant.TPM                     1.000000                   0.700195   \n",
       "H7.TPM                                 0.511230                   0.531738   \n",
       "THP1_d3_A.TPM                          0.474854                   0.989258   \n",
       "\n",
       "0                   ...  HALLMARK_COAGULATION  HALLMARK_IL2_STAT5_SIGNALING  \\\n",
       "Name                ...                                                       \n",
       "THP1_d3_B.TPM       ...              0.675781                      0.810059   \n",
       "2D_1.TPM            ...              0.025284                      0.065186   \n",
       "TRId7CWa.quant.TPM  ...              0.927246                      0.736328   \n",
       "H7.TPM              ...              0.822754                      0.816406   \n",
       "THP1_d3_A.TPM       ...              0.691406                      0.826660   \n",
       "\n",
       "0                   HALLMARK_BILE_ACID_METABOLISM  HALLMARK_PEROXISOME  \\\n",
       "Name                                                                     \n",
       "THP1_d3_B.TPM                            0.680176             0.774902   \n",
       "2D_1.TPM                                 0.221802             0.194458   \n",
       "TRId7CWa.quant.TPM                       0.484619             0.130737   \n",
       "H7.TPM                                   0.113708             0.248291   \n",
       "THP1_d3_A.TPM                            0.710938             0.877930   \n",
       "\n",
       "0                   HALLMARK_ALLOGRAFT_REJECTION  HALLMARK_SPERMATOGENESIS  \\\n",
       "Name                                                                         \n",
       "THP1_d3_B.TPM                           0.928711                  0.793457   \n",
       "2D_1.TPM                                0.019119                  0.555664   \n",
       "TRId7CWa.quant.TPM                      0.436523                  0.053558   \n",
       "H7.TPM                                  0.477051                  0.189209   \n",
       "THP1_d3_A.TPM                           0.928223                  0.791504   \n",
       "\n",
       "0                   HALLMARK_KRAS_SIGNALING  HALLMARK_KRAS_SIGNALING_UP  \\\n",
       "Name                                                                      \n",
       "THP1_d3_B.TPM                      1.000000                    0.880859   \n",
       "2D_1.TPM                           0.000000                    0.037323   \n",
       "TRId7CWa.quant.TPM                 0.354004                    0.771484   \n",
       "H7.TPM                             0.544434                    0.707520   \n",
       "THP1_d3_A.TPM                      0.947754                    0.933105   \n",
       "\n",
       "0                   HALLMARK_KRAS_SIGNALING_DN  HALLMARK_PANCREAS_BETA_CELLS  \n",
       "Name                                                                          \n",
       "THP1_d3_B.TPM                         0.337402                      0.670410  \n",
       "2D_1.TPM                              0.343262                      0.248169  \n",
       "TRId7CWa.quant.TPM                    0.968750                      0.720703  \n",
       "H7.TPM                                0.644043                      0.258057  \n",
       "THP1_d3_A.TPM                         0.476562                      0.819824  \n",
       "\n",
       "[5 rows x 3284 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prediction data \n",
    "predictData = pd.DataFrame(data = myCCLSignature).transpose()\n",
    "new_header = predictData.iloc[0] \n",
    "predictData = predictData[1:] \n",
    "predictData.columns = new_header \n",
    "predictData = predictData.apply(pd.to_numeric, errors='ignore')\n",
    "predictData = predictData.set_index(['Name'])\n",
    "predictData = normalize(predictData)\n",
    "predictData = predictData.astype('float16')\n",
    "predictData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "06a614de",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 26 entries, THP1_d3_B.TPM to HCd7CWa.quant.TPM\n",
      "Columns: 3284 entries, BIOCARTA_GRANULOCYTES_PATHWAY to HALLMARK_PANCREAS_BETA_CELLS\n",
      "dtypes: float16(3284)\n",
      "memory usage: 168.5 KB\n"
     ]
    }
   ],
   "source": [
    "predictData.info(memory_usage = 'deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c6935a1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare train set\n",
    "trainData = pd.DataFrame(data = CCLECCLSignature).transpose()\n",
    "new_header = trainData.iloc[0] \n",
    "trainData = trainData[1:] \n",
    "trainData.columns = new_header "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "52014506",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIOCARTA_GRANULOCYTES_PATHWAY</th>\n",
       "      <th>BIOCARTA_LYM_PATHWAY</th>\n",
       "      <th>BIOCARTA_BLYMPHOCYTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_CARM_ER_PATHWAY</th>\n",
       "      <th>BIOCARTA_LAIR_PATHWAY</th>\n",
       "      <th>BIOCARTA_VDR_PATHWAY</th>\n",
       "      <th>BIOCARTA_MTA3_PATHWAY</th>\n",
       "      <th>BIOCARTA_GABA_PATHWAY</th>\n",
       "      <th>BIOCARTA_EGFR_SMRTE_PATHWAY</th>\n",
       "      <th>BIOCARTA_MONOCYTE_PATHWAY</th>\n",
       "      <th>...</th>\n",
       "      <th>HALLMARK_COAGULATION</th>\n",
       "      <th>HALLMARK_IL2_STAT5_SIGNALING</th>\n",
       "      <th>HALLMARK_BILE_ACID_METABOLISM</th>\n",
       "      <th>HALLMARK_PEROXISOME</th>\n",
       "      <th>HALLMARK_ALLOGRAFT_REJECTION</th>\n",
       "      <th>HALLMARK_SPERMATOGENESIS</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_UP</th>\n",
       "      <th>HALLMARK_KRAS_SIGNALING_DN</th>\n",
       "      <th>HALLMARK_PANCREAS_BETA_CELLS</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ACH-001113</th>\n",
       "      <td>-4026.0</td>\n",
       "      <td>-2440.0</td>\n",
       "      <td>-4668.0</td>\n",
       "      <td>4244.0</td>\n",
       "      <td>-2634.00</td>\n",
       "      <td>6680.0</td>\n",
       "      <td>5124.0</td>\n",
       "      <td>233.5</td>\n",
       "      <td>3240.0</td>\n",
       "      <td>-1373.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-63.468750</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>90.1875</td>\n",
       "      <td>4124.0</td>\n",
       "      <td>-741.5</td>\n",
       "      <td>-557.0</td>\n",
       "      <td>2440.0</td>\n",
       "      <td>-862.0000</td>\n",
       "      <td>-3302.0</td>\n",
       "      <td>-1723.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000242</th>\n",
       "      <td>-3938.0</td>\n",
       "      <td>-2622.0</td>\n",
       "      <td>-4240.0</td>\n",
       "      <td>4712.0</td>\n",
       "      <td>-3190.00</td>\n",
       "      <td>7320.0</td>\n",
       "      <td>5044.0</td>\n",
       "      <td>-1760.0</td>\n",
       "      <td>3166.0</td>\n",
       "      <td>-1523.0</td>\n",
       "      <td>...</td>\n",
       "      <td>503.750000</td>\n",
       "      <td>2278.0</td>\n",
       "      <td>1100.0000</td>\n",
       "      <td>4736.0</td>\n",
       "      <td>245.5</td>\n",
       "      <td>-1528.0</td>\n",
       "      <td>3270.0</td>\n",
       "      <td>-351.2500</td>\n",
       "      <td>-3622.0</td>\n",
       "      <td>-2434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000327</th>\n",
       "      <td>-3782.0</td>\n",
       "      <td>-2314.0</td>\n",
       "      <td>-4324.0</td>\n",
       "      <td>5312.0</td>\n",
       "      <td>-2978.00</td>\n",
       "      <td>5864.0</td>\n",
       "      <td>4988.0</td>\n",
       "      <td>754.0</td>\n",
       "      <td>2148.0</td>\n",
       "      <td>-1586.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-9.976562</td>\n",
       "      <td>2174.0</td>\n",
       "      <td>1878.0000</td>\n",
       "      <td>5460.0</td>\n",
       "      <td>-374.5</td>\n",
       "      <td>-1132.0</td>\n",
       "      <td>2942.0</td>\n",
       "      <td>-475.2500</td>\n",
       "      <td>-3416.0</td>\n",
       "      <td>-998.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000461</th>\n",
       "      <td>-3736.0</td>\n",
       "      <td>-1105.0</td>\n",
       "      <td>-4948.0</td>\n",
       "      <td>4512.0</td>\n",
       "      <td>-2696.00</td>\n",
       "      <td>6556.0</td>\n",
       "      <td>4376.0</td>\n",
       "      <td>-550.0</td>\n",
       "      <td>3432.0</td>\n",
       "      <td>-1386.0</td>\n",
       "      <td>...</td>\n",
       "      <td>694.500000</td>\n",
       "      <td>2594.0</td>\n",
       "      <td>-367.0000</td>\n",
       "      <td>4192.0</td>\n",
       "      <td>313.0</td>\n",
       "      <td>-1276.0</td>\n",
       "      <td>3834.0</td>\n",
       "      <td>-95.9375</td>\n",
       "      <td>-3930.0</td>\n",
       "      <td>-1972.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACH-000792</th>\n",
       "      <td>-2076.0</td>\n",
       "      <td>1704.0</td>\n",
       "      <td>-1812.0</td>\n",
       "      <td>4140.0</td>\n",
       "      <td>448.75</td>\n",
       "      <td>6204.0</td>\n",
       "      <td>4704.0</td>\n",
       "      <td>-1424.0</td>\n",
       "      <td>2440.0</td>\n",
       "      <td>2108.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1771.000000</td>\n",
       "      <td>3038.0</td>\n",
       "      <td>-129.8750</td>\n",
       "      <td>4040.0</td>\n",
       "      <td>950.0</td>\n",
       "      <td>-1170.0</td>\n",
       "      <td>3728.0</td>\n",
       "      <td>-368.7500</td>\n",
       "      <td>-4096.0</td>\n",
       "      <td>-1965.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3284 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0           BIOCARTA_GRANULOCYTES_PATHWAY  BIOCARTA_LYM_PATHWAY  \\\n",
       "Name                                                              \n",
       "ACH-001113                        -4026.0               -2440.0   \n",
       "ACH-000242                        -3938.0               -2622.0   \n",
       "ACH-000327                        -3782.0               -2314.0   \n",
       "ACH-000461                        -3736.0               -1105.0   \n",
       "ACH-000792                        -2076.0                1704.0   \n",
       "\n",
       "0           BIOCARTA_BLYMPHOCYTE_PATHWAY  BIOCARTA_CARM_ER_PATHWAY  \\\n",
       "Name                                                                 \n",
       "ACH-001113                       -4668.0                    4244.0   \n",
       "ACH-000242                       -4240.0                    4712.0   \n",
       "ACH-000327                       -4324.0                    5312.0   \n",
       "ACH-000461                       -4948.0                    4512.0   \n",
       "ACH-000792                       -1812.0                    4140.0   \n",
       "\n",
       "0           BIOCARTA_LAIR_PATHWAY  BIOCARTA_VDR_PATHWAY  \\\n",
       "Name                                                      \n",
       "ACH-001113               -2634.00                6680.0   \n",
       "ACH-000242               -3190.00                7320.0   \n",
       "ACH-000327               -2978.00                5864.0   \n",
       "ACH-000461               -2696.00                6556.0   \n",
       "ACH-000792                 448.75                6204.0   \n",
       "\n",
       "0           BIOCARTA_MTA3_PATHWAY  BIOCARTA_GABA_PATHWAY  \\\n",
       "Name                                                       \n",
       "ACH-001113                 5124.0                  233.5   \n",
       "ACH-000242                 5044.0                -1760.0   \n",
       "ACH-000327                 4988.0                  754.0   \n",
       "ACH-000461                 4376.0                 -550.0   \n",
       "ACH-000792                 4704.0                -1424.0   \n",
       "\n",
       "0           BIOCARTA_EGFR_SMRTE_PATHWAY  BIOCARTA_MONOCYTE_PATHWAY  ...  \\\n",
       "Name                                                                ...   \n",
       "ACH-001113                       3240.0                    -1373.0  ...   \n",
       "ACH-000242                       3166.0                    -1523.0  ...   \n",
       "ACH-000327                       2148.0                    -1586.0  ...   \n",
       "ACH-000461                       3432.0                    -1386.0  ...   \n",
       "ACH-000792                       2440.0                     2108.0  ...   \n",
       "\n",
       "0           HALLMARK_COAGULATION  HALLMARK_IL2_STAT5_SIGNALING  \\\n",
       "Name                                                             \n",
       "ACH-001113            -63.468750                        2013.0   \n",
       "ACH-000242            503.750000                        2278.0   \n",
       "ACH-000327             -9.976562                        2174.0   \n",
       "ACH-000461            694.500000                        2594.0   \n",
       "ACH-000792           1771.000000                        3038.0   \n",
       "\n",
       "0           HALLMARK_BILE_ACID_METABOLISM  HALLMARK_PEROXISOME  \\\n",
       "Name                                                             \n",
       "ACH-001113                        90.1875               4124.0   \n",
       "ACH-000242                      1100.0000               4736.0   \n",
       "ACH-000327                      1878.0000               5460.0   \n",
       "ACH-000461                      -367.0000               4192.0   \n",
       "ACH-000792                      -129.8750               4040.0   \n",
       "\n",
       "0           HALLMARK_ALLOGRAFT_REJECTION  HALLMARK_SPERMATOGENESIS  \\\n",
       "Name                                                                 \n",
       "ACH-001113                        -741.5                    -557.0   \n",
       "ACH-000242                         245.5                   -1528.0   \n",
       "ACH-000327                        -374.5                   -1132.0   \n",
       "ACH-000461                         313.0                   -1276.0   \n",
       "ACH-000792                         950.0                   -1170.0   \n",
       "\n",
       "0           HALLMARK_KRAS_SIGNALING  HALLMARK_KRAS_SIGNALING_UP  \\\n",
       "Name                                                              \n",
       "ACH-001113                   2440.0                   -862.0000   \n",
       "ACH-000242                   3270.0                   -351.2500   \n",
       "ACH-000327                   2942.0                   -475.2500   \n",
       "ACH-000461                   3834.0                    -95.9375   \n",
       "ACH-000792                   3728.0                   -368.7500   \n",
       "\n",
       "0           HALLMARK_KRAS_SIGNALING_DN  HALLMARK_PANCREAS_BETA_CELLS  \n",
       "Name                                                                  \n",
       "ACH-001113                     -3302.0                       -1723.0  \n",
       "ACH-000242                     -3622.0                       -2434.0  \n",
       "ACH-000327                     -3416.0                        -998.5  \n",
       "ACH-000461                     -3930.0                       -1972.0  \n",
       "ACH-000792                     -4096.0                       -1965.0  \n",
       "\n",
       "[5 rows x 3284 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter valid ID\n",
    "validSet = set()\n",
    "for name in trainData['Name']:\n",
    "    if CCLEidToCTRPNameDict[name] in cclToAUCdict:\n",
    "        validSet.add(name)\n",
    "trainData = trainData.loc[trainData['Name'].isin(validSet)]\n",
    "trainData = trainData.reset_index(drop = True)\n",
    "trainData = trainData.set_index(['Name'])\n",
    "trainData = trainData.apply(pd.to_numeric)\n",
    "trainData = trainData.astype('float16')\n",
    "trainData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c5f3e869",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 636 entries, ACH-001113 to ACH-000052\n",
      "Columns: 3284 entries, BIOCARTA_GRANULOCYTES_PATHWAY to HALLMARK_PANCREAS_BETA_CELLS\n",
      "dtypes: float16(3284)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "trainData.info(memory_usage = 'deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "90ce4112",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not all(trainData.columns == predictData.columns):\n",
    "    raise Exception(\"Column do not match!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "da4c8b21",
   "metadata": {},
   "outputs": [],
   "source": [
    "###new\n",
    "trainData['Age'] = trainData.index.map(lambda x: metaDataDict[x][0])\n",
    "trainData['Sex'] = trainData.index.map(lambda x: metaDataDict[x][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "18ed8251",
   "metadata": {},
   "outputs": [],
   "source": [
    "#counts matter\n",
    "for geneName in candidateKeyList:\n",
    "    trainData[geneName] = trainData.index.map(lambda x: mutationDataDict[x][geneName] if x in mutationDataDict else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5770b06f",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainData = normalize(trainData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "258d3930",
   "metadata": {},
   "outputs": [],
   "source": [
    "### new end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "53f9ca95",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#labels data\n",
    "labelsDataOriginal = pd.DataFrame(columns = [sub[1] for sub in CTRPCCLAUC[7:]])\n",
    "for name in trainData.index:\n",
    "    labelsDataOriginal.loc[len(labelsDataOriginal.index)] = cclToAUCdict[CCLEidToCTRPNameDict[name]]\n",
    "labelsDataOriginal = labelsDataOriginal.set_index(trainData.index)\n",
    "labelsDataOriginal = labelsDataOriginal.astype('float16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "77d19250",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 636 entries, ACH-001113 to ACH-000052\n",
      "Columns: 481 entries, zebularine to GSK-J4\n",
      "dtypes: float16(481)\n",
      "memory usage: 639.1 KB\n"
     ]
    }
   ],
   "source": [
    "labelsDataOriginal.info(memory_usage = 'deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "651b49a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#add high priority at the begining. e.g dasatinib\n",
    "labels = list(labelsDataOriginal.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "062064dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#constants\n",
    "problem_types = ['regression'] \n",
    "eval_metrics = ['mean_squared_error']\n",
    "time_limit = 60 * 60 * 24\n",
    "tops = [100,300,600]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a029b871",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'zebularine'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/y4/nb52fxtx6g934cctxtd11mz40000gn/T/ipykernel_3926/1557473128.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m          \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mlabelData\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlabelsDataOriginal\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'zebularine'"
     ]
    }
   ],
   "source": [
    "for label in labels:\n",
    "    try:\n",
    "        if len(os.listdir(label)) > 8:\n",
    "             continue\n",
    "    except:\n",
    "        print('Working on' + label)\n",
    "    \n",
    "    labelData = labelsDataOriginal[[label]]    \n",
    "    trainDataSet = pd.concat([trainData, labelData], axis = 1)\n",
    "    \n",
    "    #first time training \n",
    "    save_path = label + '/' + 'GiloML_predictDrugAUC_Full_Feature_Medium_Quality_Model_' + label \n",
    "    multi_predictor = MultilabelPredictor(labels=[label], problem_types=problem_types, eval_metrics=eval_metrics, path=save_path)\n",
    "    multi_predictor.fit(trainDataSet, time_limit=time_limit)\n",
    "    #multi_predictor = MultilabelPredictor.load(save_path)\n",
    "    result = multi_predictor.predict(predictData)\n",
    "    result.to_csv(label + '/' + 'GiloML_predictDrugAUC_Full_Feature_Medium_Quality_Result_' + label+ '.csv')\n",
    "    \n",
    "    #get feature importance\n",
    "    predictor = multi_predictor.get_predictor(label)\n",
    "    feature_importance = predictor.feature_importance(trainDataSet[trainDataSet[label] > float('-inf')], num_shuffle_sets = 3)\n",
    "    feature_importance.to_csv(label + '/' + \"GlioML_feature_importance_\" + label + \".csv\")\n",
    "    \n",
    "    #only consider feature has positive contribution\n",
    "    feature_importance = feature_importance[feature_importance['importance'] > 0]\n",
    "    \n",
    "    #Evaluation\n",
    "    evaaluateResult = []\n",
    "    for top in tops:\n",
    "        reducedfeatures = list(feature_importance[:top].index)\n",
    "        trainDataR =  trainData[reducedfeatures]\n",
    "        trainDataSetR = pd.concat([trainDataR,labelData],axis = 1)\n",
    "        save_path_R = label + '/' + 'GlioML_ReducedFeature_' + str(top) + '_' + label + '_FeatureEvaluationModel'\n",
    "        train, evaldata = train_test_split(trainDataSetR,test_size = 0.2,random_state=1112)\n",
    "        predictor_single = MultilabelPredictor(labels=[label], problem_types=problem_types, eval_metrics=eval_metrics, path=save_path_R)\n",
    "        predictor_single.fit(train, time_limit=time_limit)\n",
    "        #predictor_single = MultilabelPredictor.load(save_path_R)\n",
    "        evaluationData = predictor_single.evaluate(evaldata)\n",
    "        evaaluateResult.append(evaluationData[label]['mean_squared_error'])\n",
    "        if(top == 100):\n",
    "            save_path_final = label + '/' + 'GiloML_predictDrugAUC_Top_100_Features_Best_Quality_Full_Data_Model_' + label\n",
    "            predictor_single = MultilabelPredictor(labels = [label], problem_types = problem_types, eval_metrics = eval_metrics, path = save_path_final)\n",
    "            predictor_single.fit(trainDataSetR, final = True, time_limit = time_limit)\n",
    "            result_final = predictor_single.predict(predictData[reducedfeatures])\n",
    "            result_final.to_csv(save_path_final + \"_Result.csv\")\n",
    "    \n",
    "    maxPerformanceFeatureCount = tops[evaaluateResult.index(max(evaaluateResult))]\n",
    "    evaaluateResult = [['Component Name','TOP100','TOP300','TOP600'], [label] + evaaluateResult]\n",
    "    with open(label + '/' + 'GlioML_ReducedFeature_evalScore_' + label + '.csv', 'w', newline='') as csvfile:\n",
    "        writer = csv.writer(csvfile)\n",
    "        writer.writerows(evaaluateResult)\n",
    "    \n",
    "    if maxPerformanceFeatureCount > 100: \n",
    "        reducedfeatures = list(feature_importance[:maxPerformanceFeatureCount].index)\n",
    "        trainDataR =  trainData[reducedfeatures]\n",
    "        trainDataSetR = pd.concat([trainDataR,labelData],axis = 1)\n",
    "        save_path_final = label + '/' + 'GiloML_predictDrugAUC_Top_' + str(maxPerformanceFeatureCount) +'_Features_Best_Quality_Full_Data_Model_' + label\n",
    "        predictor_single = MultilabelPredictor(labels = [label], problem_types = problem_types, eval_metrics = eval_metrics, path = save_path_final)\n",
    "        predictor_single.fit(trainDataSetR, final = True, time_limit = time_limit)\n",
    "        result_final = predictor_single.predict(predictData[reducedfeatures])\n",
    "        result_final.to_csv(save_path_final + \"_Result.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "48ffd795",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'zebularine'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "024e7bd4",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'zebularine'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/y4/nb52fxtx6g934cctxtd11mz40000gn/T/ipykernel_3926/886097661.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'zebularine'"
     ]
    }
   ],
   "source": [
    "os.listdir(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23eba2f7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}