20-5_GeneratingResults.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate clusterings and optimization results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because the traditional python interpreter seems rather slow compared to notebooks, we use this instead for running some generating procedures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gradutil as gu\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import logging\n",
    "import datetime\n",
    "import simplejson as json\n",
    "from time import time\n",
    "from pyomo.opt import SolverFactory\n",
    "from scipy.spatial.distance import euclidean\n",
    "from BorealWeights import BorealWeightedProblem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clustering(x, nclusts, seeds, logger=None, starttime=None):\n",
    "    res = dict()\n",
    "    for nclust in nclusts:\n",
    "        res_clust = dict()\n",
    "        for seedn in seeds:\n",
    "            c, xtoc, dist = gu.cluster(x, nclust, seedn, verbose=0)\n",
    "            res_clust[seedn] = {'c': c.tolist(),\n",
    "                                'xtoc': xtoc.tolist(),\n",
    "                                'dist': dist.tolist()}\n",
    "            if logger:\n",
    "                logger.info('Clustered to {} clusters. Seed {}'.format(nclust, seedn))\n",
    "            if starttime:\n",
    "                logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "        res[nclust] = res_clust\n",
    "        if logger:\n",
    "            logger.info('Clustered to {:2.0f} clusters'.format(nclust))\n",
    "        if starttime:\n",
    "            logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "        #with open('clusterings/new_{}.json'.format(nclust), 'w') as file:\n",
    "         #   json.dump(res_clust, file)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clustering_to_dict(readfile):\n",
    "    with open(readfile, 'r') as rfile:\n",
    "        clustering = json.loads(rfile.read())\n",
    "\n",
    "    new_clustering = dict()\n",
    "    for seedn in clustering.keys():\n",
    "        new_clustering[eval(seedn)] = dict()\n",
    "        for key in clustering[seedn].keys():\n",
    "            new_clustering[eval(seedn)][key] = np.array(clustering[seedn][key])\n",
    "    return new_clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#    def clustering_to_optims(x_orig, x_clust, x_opt, names, nclust, opt, logger=None, starttime=None):\n",
    "start = time()\n",
    "logger.info('Started optimizing')\n",
    "names = ['revenue', 'carbon', 'deadwood', 'ha']\n",
    "nclusts4 = range(1700, 8501, 200)\n",
    "x_orig = x_stack\n",
    "x_clust = x_norm\n",
    "x_opt = x_norm_stack\n",
    "starttime = start\n",
    "for nclust in nclusts4:\n",
    "        readfile = 'clusterings/new_{}.json'.format(nclust)\n",
    "        with open(readfile, 'r') as rfile:\n",
    "            read_clustering = json.loads(rfile.read())\n",
    "\n",
    "        clustering = dict()\n",
    "        for seedn in read_clustering.keys():\n",
    "            clustering[eval(seedn)] = dict()\n",
    "            for key in read_clustering[seedn].keys():\n",
    "                clustering[eval(seedn)][key] = np.array(read_clustering[seedn][key])\n",
    "        \n",
    "        n_optims = dict()\n",
    "        for seedn in clustering.keys():\n",
    "            xtoc = np.array(clustering[seedn]['xtoc'])\n",
    "            #if logger:\n",
    "            logger.info('Assigning weights')\n",
    "            #if starttime:\n",
    "            logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=int(time()-starttime)))))\n",
    "            w = np.array([sum(xtoc == i)\n",
    "                          for i in range(nclust)\n",
    "                          if sum(xtoc == i) > 0])\n",
    "            # Calculate the euclidian center of the cluster (mean)\n",
    "            # and then the point closest to that center according to\n",
    "            # euclidian distance, and then use the data format meant\n",
    "            # for optimization\n",
    "            #if logger:\n",
    "            logger.info('Assigning centers')\n",
    "            #if starttime:\n",
    "            logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=int(time()-starttime)))))\n",
    "            indices = [min(np.array(range(len(xtoc)))[xtoc == i],\n",
    "                           key=lambda index: euclidean(x_clust[index],\n",
    "                                                       np.mean(x_clust[xtoc == i],\n",
    "                                                               axis=0)))\n",
    "                       for i in range(nclust) if sum(xtoc == i) > 0]\n",
    "            c_close = x_opt[indices]\n",
    "            x_close = x_orig[indices]\n",
    "            problems = [BorealWeightedProblem(c_close[:, :, i], weights=w)\n",
    "                        for i in range(np.shape(c_close)[-1])]\n",
    "            #if logger:\n",
    "            logger.info('Solving problems')\n",
    "            #if starttime:\n",
    "            logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=int(time()-starttime)))))\n",
    "            for p in problems:\n",
    "                opt.solve(p.model)\n",
    "            n_optims[seedn] = dict()\n",
    "            for ind, name in enumerate(names):\n",
    "                n_optims[seedn][name] = dict()\n",
    "                n_optims[seedn][name]['real'] = gu.model_to_real_values(\n",
    "                    x_orig[:, :, ind],\n",
    "                    problems[ind].model,\n",
    "                    xtoc)\n",
    "                n_optims[seedn][name]['surrogate'] = gu.cluster_to_value(\n",
    "                    x_close[:, :, ind], gu.res_to_list(problems[ind].model), w)\n",
    "            #if logger:\n",
    "            logger.info('Optimized {} clusters with seed {}'.format(nclust, seedn))\n",
    "            #if starttime:\n",
    "            logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=int(time()-starttime)))))\n",
    "        #if logger:\n",
    "        logger.info('Optimized {} clusters with every seed'.format(nclust))\n",
    "        #if starttime:\n",
    "        logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=int(time()-starttime)))))\n",
    "        with open('optimizations/hope_{}.json'.format(nclust), 'w') as file:\n",
    "            json.dump(n_optims, file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "revenue, carbon, deadwood, ha = gu.init_boreal()\n",
    "\n",
    "n_revenue = gu.nan_to_bau(revenue)\n",
    "n_carbon = gu.nan_to_bau(carbon)\n",
    "n_deadwood = gu.nan_to_bau(deadwood)\n",
    "n_ha = gu.nan_to_bau(ha)\n",
    "\n",
    "revenue_norm = gu.new_normalize(n_revenue.values)\n",
    "carbon_norm = gu.new_normalize(n_carbon.values)\n",
    "deadwood_norm = gu.new_normalize(n_deadwood.values)\n",
    "ha_norm = gu.new_normalize(n_ha.values)\n",
    "\n",
    "ide = gu.ideal(False)\n",
    "nad = gu.nadir(False)\n",
    "opt = SolverFactory('cplex')\n",
    "\n",
    "x = np.concatenate((n_revenue.values, n_carbon.values, n_deadwood.values, n_ha.values), axis=1)\n",
    "x_stack = np.dstack((n_revenue, n_carbon, n_deadwood, n_ha))\n",
    "\n",
    "x_norm = np.concatenate((revenue_norm, carbon_norm, deadwood_norm, ha_norm), axis=1)\n",
    "x_norm_stack = np.dstack((revenue_norm, carbon_norm, deadwood_norm, ha_norm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(level=logging.INFO)\n",
    "logger = logging.getLogger(__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "start = time()\n",
    "logger.info('Started clustering')\n",
    "nclusts3 = range(1600, 1700, 50)\n",
    "seeds = range(2, 12)\n",
    "\n",
    "clustering(x_norm, [600], [2], logger, start)\n",
    "logger.info('All clustered to 50. Time since start {}.'.format(str(datetime.timedelta(seconds=time()-start))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "start = time()\n",
    "logger.info('Started optimizing')\n",
    "names = ['revenue', 'carbon', 'deadwood', 'ha']\n",
    "nclusts4 = range(1700, 8501, 200)\n",
    "for nclust in nclusts4:\n",
    "    clustering_to_optims(x_stack, x_norm, x_norm_stack, names, nclust, opt, logger=logger, starttime=start)\n",
    "logger.info('All optimized: 1700-8500-200. Since start {}'.format(str(datetime.timedelta(seconds=int(time()-start)))))\n"
   ]
  }
 ],
 "metadata": {
  "git": {
   "suppress_outputs": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}