internal_quality_analysis-sonarqube.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Analytics - Product Quality\n",
    "\n",
    "#### Date: 2020/02\n",
    "\n",
    "#### SUMMARY:\n",
    "\n",
    "- This notebook represents the project quality analysis of the date exposed right above. \n",
    "\n",
    "### TEAM:\n",
    "\n",
    "##### Semester: YYYY/0X\n",
    "##### Professor: Hilmer Neri\n",
    "\n",
    "##### Members:\n",
    "\n",
    "- Member x\n",
    "- Member y\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### LIBRARIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Deal with data\n",
    "import pandas as pd\n",
    "import json\n",
    "from glob import glob\n",
    "import os\n",
    "\n",
    "# Deal with visualization\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Deal with type hints\n",
    "from typing import List\n",
    "\n",
    "# Deal with time\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### GRAPH SETTINGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%config InlineBackend.figure_format ='retina'\n",
    "sns.set(font_scale=1.5)\n",
    "sns.set_style('darkgrid',\n",
    "              {'xtick.bottom' : True,\n",
    "               'ytick.left': True,\n",
    "               'grid.linestyle':'--',\n",
    "               'font.monospace': ['Computer Modern Typewriter'],\n",
    "               'axes.edgecolor' : 'white'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### DATAFRAME SETTINGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pd.set_option(\"display.max_rows\", None, \"display.max_columns\", None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Replace your semester, project name, repository name, and the programming language extension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Set your repo major name here\n",
    "# Example: fga-eps-mds-2022-1-MeasureSoftGram-\n",
    "repo_name = 'fga-eps-mds-2021-1-YourProjectName-'\n",
    "\n",
    "# Add your repos here\n",
    "# Example: 'Front': 'py',\n",
    "repos_language = {\n",
    "    'YourRepoName': 'py',\n",
    "    'YourRepoName': 'py',\n",
    "    'YourRepoName': 'ts',\n",
    "    'YourRepoName': 'py',\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### SonarCloud\n",
    "##### Path to the folder with all your jsons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Maybe you should change this path to your own path\n",
    "\n",
    "sonar_files = glob('./analytics-raw-data/fga-eps-mds-*.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Create DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Unmarshall json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def unmarshall(json_path: str) -> dict:\n",
    "    with open(json_path) as json_file:\n",
    "        json_obj = json.load(json_file)\n",
    "    return json_obj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Create a list with all valid columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "metric_list = ['files',\n",
    "               'functions',\n",
    "               'complexity',\n",
    "               'comment_lines_density',\n",
    "               'duplicated_lines_density',\n",
    "               'coverage',\n",
    "               'ncloc',\n",
    "               'tests',\n",
    "               'test_errors',\n",
    "               'test_failures',\n",
    "               'test_execution_time',\n",
    "               'security_rating']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Extract files dataframe out of component dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_files_df(df: pd.DataFrame) -> pd.DataFrame:\n",
    "\n",
    "    files_df = df[df['qualifier'] == 'FIL']\n",
    "\n",
    "    files_df = files_df.dropna(subset=['functions', 'complexity','comment_lines_density', 'duplicated_lines_density', 'coverage' ])\n",
    "\n",
    "    return files_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Extract directories dataframe out of component dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_dir_df(df: pd.DataFrame) -> pd.DataFrame:\n",
    "    dirs = df[df[\"qualifier\"] == \"DIR\"]\n",
    "\n",
    "    newdf = pd.to_numeric(dirs[\"tests\"])\n",
    "\n",
    "    max_value_index = newdf.idxmax()\n",
    "\n",
    "    return dirs.loc[max_value_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Extract uts dataframe out of component dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_uts_df(df: pd.DataFrame) -> pd.DataFrame:\n",
    "    uts_df = df[df['qualifier'] == 'UTS']\n",
    "\n",
    "    uts_df = uts_df.fillna(0)\n",
    "\n",
    "    uts_df = uts_df.dropna(subset=['test_execution_time'])\n",
    "\n",
    "    return uts_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Generate component dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def metric_per_file(json_dict: dict) -> List[dict]:\n",
    "    file_json = []\n",
    "\n",
    "    for component in json_dict['components']:\n",
    "        ncloc_value = 0\n",
    "        for measure in component['measures']:\n",
    "            if measure['metric'] == 'ncloc':\n",
    "                ncloc_value = float(measure['value'])\n",
    "                break\n",
    "\n",
    "        if (component['qualifier'] == 'FIL' and ncloc_value > 0) \\\n",
    "                or component['qualifier'] == 'DIR' \\\n",
    "                or component['qualifier'] == 'UTS':\n",
    "            file_json.append(component)\n",
    "\n",
    "    return file_json\n",
    "\n",
    "\n",
    "def generate_component_dataframe_data(\n",
    "        metrics_list: List[str],\n",
    "        file_component_data: List[dict],\n",
    "        language_extension: str) -> pd.DataFrame:\n",
    "\n",
    "    df_columns = metrics_list\n",
    "\n",
    "    files_df = pd.DataFrame(columns = df_columns)\n",
    "    dirs_df = pd.DataFrame(columns = df_columns)\n",
    "    uts_df = pd.DataFrame(columns = df_columns)\n",
    "\n",
    "    for file in file_component_data:\n",
    "        try:\n",
    "                if file['qualifier'] == 'FIL' and file['language'] == language_extension:\n",
    "                    for measure in file['measures']:\n",
    "                        files_df.at[file['path'], measure['metric']] = measure['value']\n",
    "\n",
    "                    files_df['qualifier'] = file['qualifier']\n",
    "\n",
    "                elif file['qualifier'] == 'DIR':\n",
    "                    for measure in file['measures']:\n",
    "                        dirs_df.at[file['path'], measure['metric']] = measure['value']\n",
    "\n",
    "                    dirs_df['qualifier'] = file['qualifier']\n",
    "\n",
    "                elif file['qualifier'] == 'UTS':\n",
    "                    for measure in file['measures']:\n",
    "                        uts_df.at[file['path'], measure['metric']] = measure['value']\n",
    "\n",
    "                    uts_df['qualifier'] = file['qualifier']\n",
    "\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    files_df.reset_index(inplace = True)\n",
    "    dirs_df.reset_index(inplace = True)\n",
    "    uts_df.reset_index(inplace = True)\n",
    "\n",
    "    files_df = files_df.rename({'index': 'path'}, axis=1).drop(['files'], axis=1)\n",
    "    dirs_df = dirs_df.rename({'index': 'path'}, axis=1).drop(['files'], axis=1)\n",
    "    uts_df = uts_df.rename({'index': 'path'}, axis=1).drop(['files'], axis=1)\n",
    "\n",
    "    df = pd.concat([files_df, dirs_df, uts_df], axis=0)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def create_component_df(json_list):\n",
    "    df = pd.DataFrame()\n",
    "\n",
    "    for json_path in json_list:\n",
    "        file_component = unmarshall(json_path)\n",
    "        file_component_data = metric_per_file(file_component)\n",
    "\n",
    "        base_name = os.path.basename(json_path)\n",
    "\n",
    "        file_component_dataframe = generate_component_dataframe_data(\n",
    "            metric_list,\n",
    "            file_component_data,\n",
    "            language_extension = repos_language[base_name.split(\"-\")[6]])\n",
    "\n",
    "\n",
    "        file_component_dataframe['filename'] = base_name\n",
    "\n",
    "        df = pd.concat([df, file_component_dataframe], ignore_index=True)\n",
    "\n",
    "    aux_df = df['filename'].str.split(r\"-(\\d+-\\d+-\\d+-\\d+-\\d+-\\d+)-(.*?).json\", expand=True)\n",
    "\n",
    "    df['repository'] = aux_df[0]\n",
    "    df['datetime'] = aux_df[1]\n",
    "    df['version'] = aux_df[2]\n",
    "\n",
    "    df = df.sort_values(by=['repository', 'datetime'])\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "file_component_df = create_component_df(sonar_files)\n",
    "file_component_df.repository.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Create dataframe per repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repos_dataframes = []\n",
    "\n",
    "for repo in repos_language.keys():\n",
    "    dataframe = file_component_df[file_component_df['repository'] == repo_name+repo]\n",
    "    repos_dataframes.append({'name': repo, 'df': dataframe})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Measure calculations according to Q-Rapids quality model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def _ncloc(df: pd.DataFrame) -> int:\n",
    "    ncloc = 0\n",
    "    for each in df['ncloc']:\n",
    "        # try to cast the current ncloc value to int, if the value is NaN/Null, consider it as zero.\n",
    "        try:\n",
    "            n = int(each)\n",
    "        except ValueError:\n",
    "            n = 0\n",
    "        ncloc += n\n",
    "\n",
    "    return ncloc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Quality Aspect - Maintainability"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Factor - Code Quality"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Complexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def complexity(df: pd.DataFrame):\n",
    "\n",
    "    files_df = get_files_df(df)\n",
    "\n",
    "    density_non_complex_files = len(files_df[(files_df['complexity'].astype(float) /\n",
    "                                              files_df['functions'].astype(float)) < 10]) / len(files_df)\n",
    "\n",
    "    return density_non_complex_files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def comments(df: pd.DataFrame):\n",
    "\n",
    "    files_df = get_files_df(df)\n",
    "\n",
    "    density_comment_files = len(files_df[(files_df['comment_lines_density'].astype(float) > 10) &\n",
    "                                         (files_df['comment_lines_density'].astype(float) < 30)]) / len(files_df)\n",
    "\n",
    "    return density_comment_files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Duplications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def duplication(df: pd.DataFrame):\n",
    "\n",
    "    files_df = get_files_df(df)\n",
    "\n",
    "    duplication = len(files_df[(files_df['duplicated_lines_density'].astype(float) < 5)])/len(files_df)\n",
    "\n",
    "    return duplication"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Quality Aspect - Reliability"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Factor - Testing Status"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Passed tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def test_success(df: pd.DataFrame):\n",
    "\n",
    "    dir_df = get_dir_df(df)\n",
    "\n",
    "    passed_tests = (float(dir_df['tests']) - (float(dir_df['test_errors']) + float(dir_df['test_failures']))) /\\\n",
    "               float(dir_df['tests'])\n",
    "\n",
    "    return passed_tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Fast test builds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def fast_tests(df: pd.DataFrame):\n",
    "    dir_df = get_uts_df(df)\n",
    "\n",
    "    density_fast_test_builds = len(dir_df[(dir_df['test_execution_time'].astype(float)) < 300000]) /\\\n",
    "                               len(dir_df['test_execution_time'].astype(float))\n",
    "    return density_fast_test_builds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Test coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def coverage(df: pd.DataFrame):\n",
    "\n",
    "    files_df = get_files_df(df)\n",
    "\n",
    "    density_test_coverage = len(files_df[(files_df['coverage'].astype(float) > 60)]) / len(files_df)\n",
    "\n",
    "    return density_test_coverage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Calculate measures for each repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def create_metrics_df(df: pd.DataFrame) -> pd.DataFrame:\n",
    "\n",
    "    date_time_vec = df['datetime'].unique()\n",
    "\n",
    "    m1_list = []\n",
    "    m2_list = []\n",
    "    m3_list = []\n",
    "    m4_list = []\n",
    "    m5_list = []\n",
    "    m6_list = []\n",
    "\n",
    "    ncloc_list = []\n",
    "    repository_list = []\n",
    "    version_list = []\n",
    "\n",
    "    for version in date_time_vec:\n",
    "\n",
    "        version_df = df[df['datetime'] == version]\n",
    "\n",
    "        try:\n",
    "            m1_list.append(complexity(version_df))\n",
    "        except Exception:\n",
    "            m1_list.append(0)\n",
    "\n",
    "        try:\n",
    "            m2_list.append(comments(version_df))\n",
    "        except Exception:\n",
    "            m2_list.append(0)\n",
    "\n",
    "        try:\n",
    "            m3_list.append(duplication(version_df))\n",
    "        except Exception:\n",
    "            m3_list.append(0)\n",
    "\n",
    "        try:\n",
    "            m4_list.append(test_success(version_df))\n",
    "        except Exception:\n",
    "            m4_list.append(0)\n",
    "\n",
    "        try:\n",
    "            m5_list.append(fast_tests(version_df))\n",
    "        except Exception:\n",
    "            m5_list.append(0)\n",
    "\n",
    "        try:\n",
    "            m6_list.append(coverage(version_df))\n",
    "        except Exception:\n",
    "            m6_list.append(0)\n",
    "\n",
    "        ncloc_list.append(_ncloc(version_df))\n",
    "        repository_list.append(version_df['repository'].iloc[0])\n",
    "        version_list.append(version)\n",
    "\n",
    "    final_dict = {\n",
    "        'complexity': m1_list,\n",
    "        'comments': m2_list,\n",
    "        'duplication': m3_list,\n",
    "        'test_success': m4_list,\n",
    "        'fast_tests': m5_list,\n",
    "        'coverage': m6_list,\n",
    "        'repository': repository_list,\n",
    "        'version': version_list,\n",
    "        'ncloc': ncloc_list\n",
    "    }\n",
    "\n",
    "    metrics_df = pd.DataFrame(final_dict)\n",
    "\n",
    "    return metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Here we will create a dictionary with the metrics for each repository\n",
    "metrics = {}\n",
    "\n",
    "for repo_df in repos_dataframes:\n",
    "    metrics[repo_df['name']] = create_metrics_df(repo_df['df'])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Data visualization\n",
    "\n",
    "In this area you will need to plot the metrics of each repository."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, data in metrics.items():\n",
    "    fig = plt.figure(figsize=(20, 10))\n",
    "\n",
    "    plt.plot(data['complexity'], linewidth=3, marker='<', markersize=10, label=\"complexity - Complexity\")\n",
    "    plt.plot(data['comments'], linewidth=3, marker='s', markersize=10, label=\"comments - Comments\")\n",
    "    plt.plot(data['duplication'], linewidth=3, marker='p', markersize=10, label=\"duplication - Duplication\")\n",
    "    plt.plot(data['test_success'], linewidth=3, marker='v', markersize=10, label=\"test_success - Passed tests\")\n",
    "    plt.plot(data['fast_tests'], linewidth=3, marker='^', markersize=10, label=\"fast_tests - Fast test builds\")\n",
    "    plt.plot(data['coverage'], linewidth=3, marker='>', markersize=10, label=\"coverage - Coverage\")\n",
    "\n",
    "    plt.title(f\"{name} - Metrics\", fontsize=20)\n",
    "    plt.legend(loc='best')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Quality factor and aspect aggregation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "psc1 = 1\n",
    "psc2 = 1\n",
    "pc1 = 0.5\n",
    "pc2 = 0.5\n",
    "pm1 = 0.33\n",
    "pm2 = 0.33\n",
    "pm3 = 0.33\n",
    "pm4 = 0.25\n",
    "pm5 = 0.25\n",
    "pm6 = 0.5\n",
    "\n",
    "\n",
    "# Here you will need to create the code_quality and testing_status metrics for each repository.\n",
    "\n",
    "for name, data in metrics.items():\n",
    "    data['code_quality'] = ((data['complexity']*pm1) + (data['comments']*pm2) + (data['duplication']*pm3)) * psc1\n",
    "    data['testing_status'] = ((data['test_success']*pm4) + (data['fast_tests']*pm5) + (data['coverage']*pm6)) * psc2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Code Quality visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(20, 10))\n",
    "\n",
    "for name, data in metrics.items():\n",
    "    plt.plot(data['code_quality'], linewidth=3, marker='o', markersize=10, label=name)\n",
    "\n",
    "plt.title(\"Code Quality\", fontsize=20)\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Testing Status visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(20, 10))\n",
    "\n",
    "for name, data in metrics.items():\n",
    "    plt.plot(data['testing_status'], linewidth=3, marker='o', markersize=10, label=name)\n",
    "\n",
    "plt.title(\"Testing Status\", fontsize=20)\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Aggregations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for name, data in metrics.items():\n",
    "    data['Maintainability'] = data['code_quality'] * pc1\n",
    "    data['Reliability'] = data['testing_status'] * pc2\n",
    "    data['total'] = data['Maintainability'] + data['Reliability']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Repositories analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_characteristc_stats(repo_series):\n",
    "    return {\n",
    "        'mean': repo_series.mean(),\n",
    "        'mode': repo_series.mode(),\n",
    "        'median': repo_series.median(),\n",
    "        'std': repo_series.std(),\n",
    "        'var': repo_series.var(),\n",
    "        'min': repo_series.min(),\n",
    "        'max': repo_series.max()\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analysis(metrics, name):\n",
    "    maintainability_stats = pd.DataFrame(get_characteristc_stats(metrics[\"Maintainability\"]),\n",
    "                                     columns=['mean', 'mode', 'median', 'std', 'var', 'min', 'max'])\n",
    "\n",
    "    reliability_stats = pd.DataFrame(get_characteristc_stats(metrics[\"Reliability\"]),\n",
    "                                 columns=['mean', 'mode', 'median', 'std', 'var', 'min', 'max'])\n",
    "\n",
    "\n",
    "    print(\"Maintainability Stats\")\n",
    "    print(maintainability_stats.to_string(index=False))\n",
    "\n",
    "    print(\"Reliability Stats\")\n",
    "    print(reliability_stats.to_string(index=False))\n",
    "\n",
    "    fig = plt.figure(figsize=(20, 10))\n",
    "\n",
    "    plt.plot(metrics['Maintainability'], linewidth=3, marker='o', markersize=10, label=\"Maintainability\")\n",
    "    plt.plot(metrics['Reliability'], linewidth=3, marker='*', markersize=10, label=\"Reliability\")\n",
    "\n",
    "    plt.ylim(0.1,1.1)\n",
    "    plt.title(f'{name} - Maintainability and Reliability', fontsize=20)\n",
    "    plt.legend(loc='best')\n",
    "    plt.show()\n",
    "\n",
    "    fig = plt.figure(figsize=(20, 10))\n",
    "\n",
    "    plt.plot(metrics['total'], linewidth=3, marker='X', markersize=5)\n",
    "\n",
    "    plt.ylim(0.1,1.1)\n",
    "    plt.title(f'{name} - Total', fontsize=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Analysis loop in each repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for name, data in metrics.items():\n",
    "    print(name)\n",
    "    analysis(data, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality characteristic indicator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for name, data in metrics.items():\n",
    "    fig = plt.figure(figsize=(20, 10))\n",
    "    sns.boxplot(data=data[['Maintainability','Reliability']])\n",
    "\n",
    "    plt.title(f\"{name} - Quality characteristic indicator\", fontsize=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Quality indicator visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(20, 10))\n",
    "\n",
    "for name, data in metrics.items():\n",
    "    plt.plot(data['total'], linewidth=3, marker='o', markersize=10, label=name)\n",
    "\n",
    "plt.ylim(.1,1)\n",
    "plt.title(\"Total - Quality indicator\", fontsize=20)\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Export data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "metrics_list = metrics.values()\n",
    "\n",
    "metrics_df = pd.concat(metrics_list, ignore_index=True)\n",
    "\n",
    "display(metrics_df)\n",
    "\n",
    "current_datetime = datetime.datetime.now().strftime(\"%m-%d-%Y--%H-%M-%S\")\n",
    "\n",
    "metrics_df.to_excel('./data/fga-eps-mds-2022-2-MeasureSoftGram--{}.xlsx'.format(current_datetime), index = False)\n",
    "\n",
    "metrics_df.to_csv('./data/fga-eps-mds-2022-2-MeasureSoftGram--{}.csv'.format(current_datetime), index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "af8259ad5c1c9c7a69bd6ea085234cf8fd3a6a37a71ca551828b314c4d89b0ad"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}