20-5_GeneratingResults.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate clusterings and optimization results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because the traditional python interpreter seems rather slow compared to notebooks, we use this instead for running some generating procedures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gradutil as gu\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import logging\n",
    "import datetime\n",
    "import simplejson as json\n",
    "from time import time\n",
    "from pyomo.opt import SolverFactory\n",
    "from scipy.spatial.distance import euclidean\n",
    "from BorealWeights import BorealWeightedProblem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clustering(x, nclusts, seeds, logger=None, starttime=None):\n",
    "    res = dict()\n",
    "    for nclust in nclusts:\n",
    "        res_clust = dict()\n",
    "        for seedn in seeds:\n",
    "            c, xtoc, dist = gu.cluster(x, nclust, seedn, verbose=0)\n",
    "            res_clust[seedn] = {'c': c.tolist(),\n",
    "                                'xtoc': xtoc.tolist(),\n",
    "                                'dist': dist.tolist()}\n",
    "            if logger:\n",
    "                logger.info('Clustered to {} clusters. Seed {}'.format(nclust, seedn))\n",
    "            if starttime:\n",
    "                logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "        res[nclust] = res_clust\n",
    "        if logger:\n",
    "            logger.info('Clustered to {:2.0f} clusters'.format(nclust))\n",
    "        if starttime:\n",
    "            logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "        with open('clusterings/new_{}.json'.format(nclust), 'w') as file:\n",
    "            json.dump(res_clust, file)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clustering_to_dict(readfile):\n",
    "    with open(readfile, 'r') as rfile:\n",
    "        clustering = json.loads(rfile.read())\n",
    "\n",
    "    new_clustering = dict()\n",
    "    for seedn in clustering.keys():\n",
    "        new_clustering[eval(seedn)] = dict()\n",
    "        for key in clustering[seedn].keys():\n",
    "            new_clustering[eval(seedn)][key] = np.array(clustering[seedn][key])\n",
    "    return new_clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clustering_to_optims(x_orig, x_clust, x_opt, names, clustering, opt, logger=None, starttime=None):\n",
    "    #optims = dict()\n",
    "    for nclust in sorted(clustering.keys()):\n",
    "        n_optims = dict()\n",
    "        for seedn in clustering[nclust].keys():\n",
    "            xtoc = np.array(clustering[nclust][seedn]['xtoc'])\n",
    "            if logger:\n",
    "                logger.info('Assigning weights')\n",
    "            if starttime:\n",
    "                logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "            w = np.array([sum(xtoc == i)\n",
    "                          for i in range(nclust)\n",
    "                          if sum(xtoc == i) > 0])\n",
    "            # Calculate the euclidian center of the cluster (mean)\n",
    "            # and then the point closest to that center according to\n",
    "            # euclidian distance, and then use the data format meant\n",
    "            # for optimization\n",
    "            if logger:\n",
    "                logger.info('Assigning centers')\n",
    "            if starttime:\n",
    "                logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "            c_close = np.array([x_opt[min(np.array(range(len(xtoc)))[xtoc == i], \n",
    "                                          key=lambda index: euclidean(x_clust[index], \n",
    "                                                                      np.mean(x_clust[xtoc == i], \n",
    "                                                                              axis=0)))] \n",
    "                                for i in range(nclust) if sum(xtoc == i) > 0])\n",
    "            problems = [BorealWeightedProblem(c_close[:, :, i], weights=w)\n",
    "                        for i in range(np.shape(c_close)[-1])]\n",
    "            if logger:\n",
    "                logger.info('Solving problems')\n",
    "            if starttime:\n",
    "                logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "            for p in problems:\n",
    "                opt.solve(p.model)\n",
    "            n_optims[seedn] = dict()\n",
    "            for ind, name in enumerate(names):\n",
    "                n_optims[seedn][name] = dict()\n",
    "                n_optims[seedn][name]['real'] = gu.model_to_real_values(\n",
    "                    x_orig[:, :, ind],\n",
    "                    problems[ind].model,\n",
    "                    xtoc)\n",
    "                n_optims[seedn][name]['surrogate'] = gu.cluster_to_value(\n",
    "                    x_orig[:, :, ind], gu.res_to_list(problems[ind].model), w)\n",
    "            if logger:\n",
    "                logger.info('Optimized {} clusters with seed {}'.format(nclust, seedn))\n",
    "            if starttime:\n",
    "                logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "        #optims[nclust] = n_optims\n",
    "        if logger:\n",
    "            logger.info('Optimized {} clusters with every seed'.format(nclust))\n",
    "        if starttime:\n",
    "            logger.info('Since start {}.'.format(str(datetime.timedelta(seconds=time()-starttime))))\n",
    "        with open('optimizations/new_{}.json'.format(nclust), 'w') as file:\n",
    "            json.dump(n_optims, file)\n",
    "    #return optims\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "revenue, carbon, deadwood, ha = gu.init_boreal()\n",
    "\n",
    "n_revenue = gu.nan_to_bau(revenue)\n",
    "n_carbon = gu.nan_to_bau(carbon)\n",
    "n_deadwood = gu.nan_to_bau(deadwood)\n",
    "n_ha = gu.nan_to_bau(ha)\n",
    "\n",
    "revenue_norm = gu.new_normalize(n_revenue.values)\n",
    "carbon_norm = gu.new_normalize(n_carbon.values)\n",
    "deadwood_norm = gu.new_normalize(n_deadwood.values)\n",
    "ha_norm = gu.new_normalize(n_ha.values)\n",
    "\n",
    "ide = gu.ideal(False)\n",
    "nad = gu.nadir(False)\n",
    "opt = SolverFactory('cplex')\n",
    "\n",
    "x = np.concatenate((n_revenue.values, n_carbon.values, n_deadwood.values, n_ha.values), axis=1)\n",
    "x_stack = np.dstack((n_revenue, n_carbon, n_deadwood, n_ha))\n",
    "\n",
    "x_norm = np.concatenate((revenue_norm, carbon_norm, deadwood_norm, ha_norm), axis=1)\n",
    "x_norm_stack = np.dstack((revenue_norm, carbon_norm, deadwood_norm, ha_norm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(level=logging.INFO)\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "clustering_file = 'clusterings/new_all8301.json'\n",
    "optim_file = 'optimizations/new_all8301.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:__main__:Started clustering\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 2\n",
      "INFO:__main__:Since start 0:00:39.968162.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 3\n",
      "INFO:__main__:Since start 0:01:27.474076.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 4\n",
      "INFO:__main__:Since start 0:02:11.982396.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 5\n",
      "INFO:__main__:Since start 0:02:53.430474.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 6\n",
      "INFO:__main__:Since start 0:03:42.224782.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 7\n",
      "INFO:__main__:Since start 0:04:26.090134.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 8\n",
      "INFO:__main__:Since start 0:05:07.415333.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 9\n",
      "INFO:__main__:Since start 0:05:47.800097.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 10\n",
      "INFO:__main__:Since start 0:06:27.807558.\n",
      "INFO:__main__:Clustered to 1600 clusters. Seed 11\n",
      "INFO:__main__:Since start 0:07:06.482839.\n",
      "INFO:__main__:Clustered to 1600 clusters\n",
      "INFO:__main__:Since start 0:07:06.488547.\n",
      "INFO:__main__:All clustered to 1600-1700-200. Time since start 0:07:11.000639.\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "logger.info('Started clustering')\n",
    "nclusts3 = range(1600, 1700, 50)\n",
    "seeds = range(2, 12)\n",
    "\n",
    "clustering(x_norm, nclusts3, seeds, logger, start)\n",
    "logger.info('All clustered to 1600-1700-50. Time since start {}.'.format(str(datetime.timedelta(seconds=time()-start))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:__main__:Started optimizing\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 0:00:04.258278.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 0:05:35.965548.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 0:08:34.284960.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 2\n",
      "INFO:__main__:Since start 0:08:38.234586.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 0:08:38.235894.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 0:14:20.047824.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 0:17:25.739500.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 3\n",
      "INFO:__main__:Since start 0:17:29.014357.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 0:17:29.015936.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 0:23:23.609925.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 0:26:25.783004.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 4\n",
      "INFO:__main__:Since start 0:26:29.112173.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 0:26:29.114904.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 0:32:07.476445.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 0:35:24.009138.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 5\n",
      "INFO:__main__:Since start 0:35:27.447603.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 0:35:27.452571.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 0:41:39.174989.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 0:44:56.464969.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 6\n",
      "INFO:__main__:Since start 0:45:00.052282.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 0:45:00.058692.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 0:51:00.627261.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 0:54:07.208960.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 7\n",
      "INFO:__main__:Since start 0:54:10.671819.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 0:54:10.673177.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 0:59:43.400140.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 1:02:35.333071.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 8\n",
      "INFO:__main__:Since start 1:02:38.731481.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 1:02:38.733952.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 1:08:14.513054.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 1:11:10.997940.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 9\n",
      "INFO:__main__:Since start 1:11:14.332719.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 1:11:14.334415.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 5:01:17.192964.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 5:05:48.351006.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 10\n",
      "INFO:__main__:Since start 5:05:53.111197.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 5:05:53.116829.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 5:14:00.988584.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 5:16:53.361871.\n",
      "INFO:__main__:Optimized 1600 clusters with seed 11\n",
      "INFO:__main__:Since start 5:16:56.706096.\n",
      "INFO:__main__:Optimized 1600 clusters with every seed\n",
      "INFO:__main__:Since start 5:16:56.709552.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 5:17:01.061670.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 5:22:25.709947.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 5:25:21.068728.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 2\n",
      "INFO:__main__:Since start 5:25:24.113381.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 5:25:24.119436.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 5:30:52.440008.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 5:33:50.371913.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 3\n",
      "INFO:__main__:Since start 5:33:53.835350.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 5:33:53.839416.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 5:39:22.248478.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 5:42:17.482496.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 4\n",
      "INFO:__main__:Since start 5:42:20.815838.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 5:42:20.819815.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 5:47:49.852338.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 5:50:44.351454.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 5\n",
      "INFO:__main__:Since start 5:50:47.540960.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 5:50:47.543469.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 5:56:53.860806.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 6:00:15.857303.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 6\n",
      "INFO:__main__:Since start 6:00:19.490882.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 6:00:19.492967.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 6:06:35.994801.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 6:09:57.840466.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 7\n",
      "INFO:__main__:Since start 6:10:01.576061.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 6:10:01.577829.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 6:16:12.072945.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 6:19:30.345192.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 8\n",
      "INFO:__main__:Since start 6:19:33.779792.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 6:19:33.781493.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 6:25:29.408201.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 20:58:38.928806.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 9\n",
      "INFO:__main__:Since start 20:58:42.642835.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 20:58:42.644207.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 21:04:16.743940.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 21:07:13.367789.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 10\n",
      "INFO:__main__:Since start 21:07:16.747878.\n",
      "INFO:__main__:Assigning weights\n",
      "INFO:__main__:Since start 21:07:16.751888.\n",
      "INFO:__main__:Assigning centers\n",
      "INFO:__main__:Since start 21:12:43.795456.\n",
      "INFO:__main__:Solving problems\n",
      "INFO:__main__:Since start 21:15:38.808372.\n",
      "INFO:__main__:Optimized 1650 clusters with seed 11\n",
      "INFO:__main__:Since start 21:15:42.138518.\n",
      "INFO:__main__:Optimized 1650 clusters with every seed\n",
      "INFO:__main__:Since start 21:15:42.143684.\n",
      "INFO:__main__:All optimized to 1550-1700-100. Since start 21:15:42.150064\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "logger.info('Started optimizing')\n",
    "names = ['revenue', 'carbon', 'deadwood', 'ha']\n",
    "nclusts4 = range(1600, 1700, 50)\n",
    "for nclust in nclusts4:\n",
    "    clusters = clustering_to_dict('clusterings/new_{}.json'.format(nclust))\n",
    "    clusteri = {nclust:clusters}\n",
    "    clustering_to_optims(x_stack, x_norm, x_norm_stack, names, clusteri, opt, logger=logger, starttime=start)\n",
    "logger.info('All optimized to 1550-1700-100. Since start {}'.format(str(datetime.timedelta(seconds=time()-start))))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# In the interactive optimization the given reference point has to be scaled to 0-1 also!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}