VJ_XLNet_toxic_comment_classification_challenge_share.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "VJ - XLNet_toxic_comment_classification_challenge_share.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "sHt9MdUoOaLl"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c65c8cf24e0648428cf67ba07a402ef9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5f173bf1ae734d03812be012a04df202",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3b3d2949115c44f7b3394563f403804a",
              "IPY_MODEL_f20a2ce4f1b74795a5ec648527f32aa6"
            ]
          }
        },
        "5f173bf1ae734d03812be012a04df202": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3b3d2949115c44f7b3394563f403804a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d1d79b039e96487b8248eace5bad9ee6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 798011,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 798011,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9d30d96435784558897b7207f61e6899"
          }
        },
        "f20a2ce4f1b74795a5ec648527f32aa6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b17ce5b4f3b84c3387b654196f5690e4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 798k/798k [00:15&lt;00:00, 52.1kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_21fa820e75df4b42ba8ea9ec4fc2b56d"
          }
        },
        "d1d79b039e96487b8248eace5bad9ee6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9d30d96435784558897b7207f61e6899": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b17ce5b4f3b84c3387b654196f5690e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "21fa820e75df4b42ba8ea9ec4fc2b56d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "762e096d83ad4d2da1a1eabdf1a480a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f7db7146feea443aa1a38752b2d70f42",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a44d453a84f74f3fbcbd0314190042fa",
              "IPY_MODEL_8ad076b945744438824f89a0a26a7dac"
            ]
          }
        },
        "f7db7146feea443aa1a38752b2d70f42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a44d453a84f74f3fbcbd0314190042fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f81ac122d5b24bbd8f0dc17cf510af62",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 760,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 760,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cb24bbe04e1948d58a6ffd029ac7a1d6"
          }
        },
        "8ad076b945744438824f89a0a26a7dac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dfa54375d80949e18c53241f8d7a95bc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 760/760 [00:10&lt;00:00, 72.0B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a7a8dcba4f4e47b6b61d633dcb9fd892"
          }
        },
        "f81ac122d5b24bbd8f0dc17cf510af62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cb24bbe04e1948d58a6ffd029ac7a1d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dfa54375d80949e18c53241f8d7a95bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a7a8dcba4f4e47b6b61d633dcb9fd892": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5924d3539110486596cc56ab06983076": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_84168ce181c24477af61042b482c4297",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_edc25cd550f84e8488721f4960df1b0c",
              "IPY_MODEL_87266ed5b5f747299c8058fdcbb097f0"
            ]
          }
        },
        "84168ce181c24477af61042b482c4297": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "edc25cd550f84e8488721f4960df1b0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_613671e974824cfb97197e420dd05e72",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 467042463,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 467042463,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_61f759d794d7451bb80cc380540fd35e"
          }
        },
        "87266ed5b5f747299c8058fdcbb097f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1a126d61d10b4e3282c5c3616da2716f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 467M/467M [00:10&lt;00:00, 45.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3f8f4bb9ae634340ae7c05b9124cf302"
          }
        },
        "613671e974824cfb97197e420dd05e72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "61f759d794d7451bb80cc380540fd35e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a126d61d10b4e3282c5c3616da2716f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3f8f4bb9ae634340ae7c05b9124cf302": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/vinejain/multilingual_feedback_multilabel_classification/blob/master/VJ_XLNet_toxic_comment_classification_challenge_share.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YCyGoBDWSn27",
        "colab_type": "text"
      },
      "source": [
        "REFERENCE: https://towardsdatascience.com/multi-label-text-classification-with-xlnet-b5f5755302df"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aUzQGX2ILqMK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 588
        },
        "outputId": "878d4132-fc40-43fb-e59c-904b91cc3e0f"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
            "\u001b[K     |████████████████████████████████| 778kB 2.7MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 12.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers==0.8.1.rc1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 12.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 41.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=b6341fc04e96fb58f95fa359bea104ddc23ceef796d93dd495ed031e100f0aa2\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NwTHAUqXk7_B",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "60ddd741-affd-49a0-8d4b-c898f42d3b5f"
      },
      "source": [
        "import os\n",
        "import math\n",
        "\n",
        "import torch\n",
        "from torch.nn import BCEWithLogitsLoss\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import AdamW, XLNetTokenizer, XLNetModel, XLNetLMHeadModel, XLNetConfig\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm, trange\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "print(\"GPU Available: {}\".format(torch.cuda.is_available()))\n",
        "n_gpu = torch.cuda.device_count()\n",
        "print(\"Number of GPU Available: {}\".format(n_gpu))\n",
        "print(\"GPU: {}\".format(torch.cuda.get_device_name(0)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU Available: True\n",
            "Number of GPU Available: 1\n",
            "GPU: Tesla K80\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vothBTEKll78",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FZbLvzAKlmsP",
        "colab_type": "text"
      },
      "source": [
        "## Import Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IlQpf1MmlqJI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "d1a20c65-7400-4240-c604-4d203d6b9c73"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a2Xf8DAOl0XH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train = pd.read_csv('/content/drive/My Drive/Data/toxic_comment_classification_challenge/train.csv', index_col='id')\n",
        "test = pd.read_csv('/content/drive/My Drive/Data/toxic_comment_classification_challenge/test.csv', index_col='id')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ZEGGs69tD-3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# get small sample of train to try\n",
        "train = train.sample(n=100)\n",
        "test = test.sample(n=100)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MN8bQOt2mVc5",
        "colab_type": "text"
      },
      "source": [
        "## Preprocess Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cz44ag9VMutg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_sentence_embeddings_length(text_list, tokenizer):\n",
        "    tokenized_texts = list(map(lambda t: tokenizer.tokenize(t), text_list))\n",
        "    tokenized_texts_len = list(map(lambda t: len(t), tokenized_texts))\n",
        "    fig, ax = plt.subplots(figsize=(8, 5));\n",
        "    ax.hist(tokenized_texts_len, bins=40);\n",
        "    ax.set_xlabel(\"Length of Comment Embeddings\");\n",
        "    ax.set_ylabel(\"Number of Comments\");\n",
        "    return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PTOd-qKQMyFO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "c65c8cf24e0648428cf67ba07a402ef9",
            "5f173bf1ae734d03812be012a04df202",
            "3b3d2949115c44f7b3394563f403804a",
            "f20a2ce4f1b74795a5ec648527f32aa6",
            "d1d79b039e96487b8248eace5bad9ee6",
            "9d30d96435784558897b7207f61e6899",
            "b17ce5b4f3b84c3387b654196f5690e4",
            "21fa820e75df4b42ba8ea9ec4fc2b56d"
          ]
        },
        "outputId": "aa6df9c4-2bdf-442c-cb11-91d8ba9b186e"
      },
      "source": [
        "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c65c8cf24e0648428cf67ba07a402ef9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=798011.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D3Xq-qP54SiF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_text_list = train[\"comment_text\"].values\n",
        "test_text_list = test[\"comment_text\"].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ocveMTIfM5cP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tokenize_inputs(text_list, tokenizer, num_embeddings=512):\n",
        "    \"\"\"\n",
        "    Tokenizes the input text input into ids. Appends the appropriate special\n",
        "    characters to the end of the text to denote end of sentence. Truncate or pad\n",
        "    the appropriate sequence length.\n",
        "    \"\"\"\n",
        "    # tokenize the text, then truncate sequence to the desired length minus 2 for\n",
        "    # the 2 special characters\n",
        "    tokenized_texts = list(map(lambda t: tokenizer.tokenize(t)[:num_embeddings-2], text_list))\n",
        "    # convert tokenized text into numeric ids for the appropriate LM\n",
        "    input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "    # append special token \"<s>\" and </s> to end of sentence\n",
        "    input_ids = [tokenizer.build_inputs_with_special_tokens(x) for x in input_ids]\n",
        "    # pad sequences\n",
        "    input_ids = pad_sequences(input_ids, maxlen=num_embeddings, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "    return input_ids\n",
        "\n",
        "def create_attn_masks(input_ids):\n",
        "    \"\"\"\n",
        "    Create attention masks to tell model whether attention should be applied to\n",
        "    the input id tokens. Do not want to perform attention on padding tokens.\n",
        "    \"\"\"\n",
        "    # Create attention masks\n",
        "    attention_masks = []\n",
        "\n",
        "    # Create a mask of 1s for each token followed by 0s for padding\n",
        "    for seq in input_ids:\n",
        "        seq_mask = [float(i>0) for i in seq]\n",
        "        attention_masks.append(seq_mask)\n",
        "    return attention_masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p2ZVLj_OM8sV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create input id tokens\n",
        "train_input_ids = tokenize_inputs(train_text_list, tokenizer, num_embeddings=250)\n",
        "# create input id tokens\n",
        "test_input_ids = tokenize_inputs(test_text_list, tokenizer, num_embeddings=250)\n",
        "# create attention masks\n",
        "train_attention_masks = create_attn_masks(train_input_ids)\n",
        "test_attention_masks = create_attn_masks(test_input_ids)\n",
        "# add input ids and attention masks to the dataframe\n",
        "train[\"features\"] = train_input_ids.tolist()\n",
        "train[\"masks\"] = train_attention_masks\n",
        "\n",
        "test[\"features\"] = test_input_ids.tolist()\n",
        "test[\"masks\"] = test_attention_masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uRpKDj-WN2tN",
        "colab_type": "text"
      },
      "source": [
        "## Train, Valid Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H2A1pqzcN_Gr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# train valid split\n",
        "train, valid = train_test_split(train, test_size=0.2, random_state=42)\n",
        "\n",
        "X_train = train[\"features\"].values.tolist()\n",
        "X_valid = valid[\"features\"].values.tolist()\n",
        "\n",
        "train_masks = train[\"masks\"].values.tolist()\n",
        "valid_masks = valid[\"masks\"].values.tolist()\n",
        "\n",
        "label_cols = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
        "Y_train = train[label_cols].values.tolist()\n",
        "Y_valid = valid[label_cols].values.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GQUZkER-OCVE",
        "colab_type": "text"
      },
      "source": [
        "## Create Dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jChp-AsjOBrv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Convert all of our input ids and attention masks into \n",
        "# torch tensors, the required datatype for our model\n",
        "\n",
        "X_train = torch.tensor(X_train)\n",
        "X_valid = torch.tensor(X_valid)\n",
        "\n",
        "Y_train = torch.tensor(Y_train, dtype=torch.float32)\n",
        "Y_valid = torch.tensor(Y_valid, dtype=torch.float32)\n",
        "\n",
        "train_masks = torch.tensor(train_masks, dtype=torch.long)\n",
        "valid_masks = torch.tensor(valid_masks, dtype=torch.long)\n",
        "\n",
        "# Select a batch size for training\n",
        "batch_size = 16\n",
        "\n",
        "# Create an iterator of our data with torch DataLoader. This helps save on \n",
        "# memory during training because, unlike a for loop, \n",
        "# with an iterator the entire dataset does not need to be loaded into memory\n",
        "\n",
        "train_data = TensorDataset(X_train, train_masks, Y_train)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data,\\\n",
        "                              sampler=train_sampler,\\\n",
        "                              batch_size=batch_size)\n",
        "\n",
        "validation_data = TensorDataset(X_valid, valid_masks, Y_valid)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data,\\\n",
        "                                   sampler=validation_sampler,\\\n",
        "                                   batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XCE_3AnLOPpI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(model, num_epochs,\\\n",
        "          optimizer,\\\n",
        "          train_dataloader, valid_dataloader,\\\n",
        "          model_save_path,\\\n",
        "          train_loss_set=[], valid_loss_set = [],\\\n",
        "          lowest_eval_loss=None, start_epoch=0,\\\n",
        "          device=\"cpu\"\n",
        "          ):\n",
        "  \"\"\"\n",
        "  Train the model and save the model with the lowest validation loss\n",
        "  \"\"\"\n",
        "\n",
        "  model.to(device)\n",
        "\n",
        "  # trange is a tqdm wrapper around the normal python range\n",
        "  for i in trange(num_epochs, desc=\"Epoch\"):\n",
        "    # if continue training from saved model\n",
        "    actual_epoch = start_epoch + i\n",
        "\n",
        "    # Training\n",
        "\n",
        "    # Set our model to training mode (as opposed to evaluation mode)\n",
        "    model.train()\n",
        "\n",
        "    # Tracking variables\n",
        "    tr_loss = 0\n",
        "    num_train_samples = 0\n",
        "\n",
        "    # Train the data for one epoch\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "      # Add batch to GPU\n",
        "      batch = tuple(t.to(device) for t in batch)\n",
        "      # Unpack the inputs from our dataloader\n",
        "      b_input_ids, b_input_mask, b_labels = batch\n",
        "      # Clear out the gradients (by default they accumulate)\n",
        "      optimizer.zero_grad()\n",
        "      # Forward pass\n",
        "      loss = model(b_input_ids, attention_mask=b_input_mask, labels=b_labels)\n",
        "      # store train loss\n",
        "      tr_loss += loss.item()\n",
        "      num_train_samples += b_labels.size(0)\n",
        "      # Backward pass\n",
        "      loss.backward()\n",
        "      # Update parameters and take a step using the computed gradient\n",
        "      optimizer.step()\n",
        "      #scheduler.step()\n",
        "\n",
        "    # Update tracking variables\n",
        "    epoch_train_loss = tr_loss/num_train_samples\n",
        "    train_loss_set.append(epoch_train_loss)\n",
        "\n",
        "    print(\"Train loss: {}\".format(epoch_train_loss))\n",
        "\n",
        "    # Validation\n",
        "\n",
        "    # Put model in evaluation mode to evaluate loss on the validation set\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    eval_loss = 0\n",
        "    num_eval_samples = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in valid_dataloader:\n",
        "      # Add batch to GPU\n",
        "      batch = tuple(t.to(device) for t in batch)\n",
        "      # Unpack the inputs from our dataloader\n",
        "      b_input_ids, b_input_mask, b_labels = batch\n",
        "      # Telling the model not to compute or store gradients,\n",
        "      # saving memory and speeding up validation\n",
        "      with torch.no_grad():\n",
        "        # Forward pass, calculate validation loss\n",
        "        loss = model(b_input_ids, attention_mask=b_input_mask, labels=b_labels)\n",
        "        # store valid loss\n",
        "        eval_loss += loss.item()\n",
        "        num_eval_samples += b_labels.size(0)\n",
        "\n",
        "    epoch_eval_loss = eval_loss/num_eval_samples\n",
        "    valid_loss_set.append(epoch_eval_loss)\n",
        "\n",
        "    print(\"Valid loss: {}\".format(epoch_eval_loss))\n",
        "\n",
        "    if lowest_eval_loss == None:\n",
        "      lowest_eval_loss = epoch_eval_loss\n",
        "      # save model\n",
        "      save_model(model, model_save_path, actual_epoch,\\\n",
        "                 lowest_eval_loss, train_loss_set, valid_loss_set)\n",
        "    else:\n",
        "      if epoch_eval_loss < lowest_eval_loss:\n",
        "        lowest_eval_loss = epoch_eval_loss\n",
        "        # save model\n",
        "        save_model(model, model_save_path, actual_epoch,\\\n",
        "                   lowest_eval_loss, train_loss_set, valid_loss_set)\n",
        "    print(\"\\n\")\n",
        "\n",
        "  return model, train_loss_set, valid_loss_set\n",
        "\n",
        "\n",
        "def save_model(model, save_path, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist):\n",
        "  \"\"\"\n",
        "  Save the model to the path directory provided\n",
        "  \"\"\"\n",
        "  model_to_save = model.module if hasattr(model, 'module') else model\n",
        "  checkpoint = {'epochs': epochs, \\\n",
        "                'lowest_eval_loss': lowest_eval_loss,\\\n",
        "                'state_dict': model_to_save.state_dict(),\\\n",
        "                'train_loss_hist': train_loss_hist,\\\n",
        "                'valid_loss_hist': valid_loss_hist\n",
        "               }\n",
        "  torch.save(checkpoint, save_path)\n",
        "  print(\"Saving model at epoch {} with validation loss of {}\".format(epochs,\\\n",
        "                                                                     lowest_eval_loss))\n",
        "  return\n",
        "  \n",
        "def load_model(save_path):\n",
        "  \"\"\"\n",
        "  Load the model from the path directory provided\n",
        "  \"\"\"\n",
        "  checkpoint = torch.load(save_path)\n",
        "  model_state_dict = checkpoint['state_dict']\n",
        "  model = XLNetForMultiLabelSequenceClassification(num_labels=model_state_dict[\"classifier.weight\"].size()[0])\n",
        "  model.load_state_dict(model_state_dict)\n",
        "\n",
        "  epochs = checkpoint[\"epochs\"]\n",
        "  lowest_eval_loss = checkpoint[\"lowest_eval_loss\"]\n",
        "  train_loss_hist = checkpoint[\"train_loss_hist\"]\n",
        "  valid_loss_hist = checkpoint[\"valid_loss_hist\"]\n",
        "  \n",
        "  return model, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OoxSrbFDOUW0",
        "colab_type": "text"
      },
      "source": [
        "## Train Model from Scratch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wGMnhaIuUbXx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.cuda.empty_cache()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8VMI7VWpOKj0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 114,
          "referenced_widgets": [
            "762e096d83ad4d2da1a1eabdf1a480a9",
            "f7db7146feea443aa1a38752b2d70f42",
            "a44d453a84f74f3fbcbd0314190042fa",
            "8ad076b945744438824f89a0a26a7dac",
            "f81ac122d5b24bbd8f0dc17cf510af62",
            "cb24bbe04e1948d58a6ffd029ac7a1d6",
            "dfa54375d80949e18c53241f8d7a95bc",
            "a7a8dcba4f4e47b6b61d633dcb9fd892",
            "5924d3539110486596cc56ab06983076",
            "84168ce181c24477af61042b482c4297",
            "edc25cd550f84e8488721f4960df1b0c",
            "87266ed5b5f747299c8058fdcbb097f0",
            "613671e974824cfb97197e420dd05e72",
            "61f759d794d7451bb80cc380540fd35e",
            "1a126d61d10b4e3282c5c3616da2716f",
            "3f8f4bb9ae634340ae7c05b9124cf302"
          ]
        },
        "outputId": "dbb48f34-0ff7-4e35-d09b-9ab3c5d466cf"
      },
      "source": [
        "#config = XLNetConfig()\n",
        "        \n",
        "class XLNetForMultiLabelSequenceClassification(torch.nn.Module):\n",
        "  \n",
        "  def __init__(self, num_labels=2):\n",
        "    super(XLNetForMultiLabelSequenceClassification, self).__init__()\n",
        "    self.num_labels = num_labels\n",
        "    self.xlnet = XLNetModel.from_pretrained('xlnet-base-cased')\n",
        "    self.classifier = torch.nn.Linear(768, num_labels)\n",
        "\n",
        "    torch.nn.init.xavier_normal_(self.classifier.weight)\n",
        "\n",
        "  def forward(self, input_ids, token_type_ids=None,\\\n",
        "              attention_mask=None, labels=None):\n",
        "    # last hidden layer\n",
        "    last_hidden_state = self.xlnet(input_ids=input_ids,\\\n",
        "                                   attention_mask=attention_mask,\\\n",
        "                                   token_type_ids=token_type_ids)\n",
        "    # pool the outputs into a mean vector\n",
        "    mean_last_hidden_state = self.pool_hidden_state(last_hidden_state)\n",
        "    logits = self.classifier(mean_last_hidden_state)\n",
        "        \n",
        "    if labels is not None:\n",
        "      loss_fct = BCEWithLogitsLoss()\n",
        "      loss = loss_fct(logits.view(-1, self.num_labels),\\\n",
        "                      labels.view(-1, self.num_labels))\n",
        "      return loss\n",
        "    else:\n",
        "      return logits\n",
        "    \n",
        "  def freeze_xlnet_decoder(self):\n",
        "    \"\"\"\n",
        "    Freeze XLNet weight parameters. They will not be updated during training.\n",
        "    \"\"\"\n",
        "    for param in self.xlnet.parameters():\n",
        "      param.requires_grad = False\n",
        "    \n",
        "  def unfreeze_xlnet_decoder(self):\n",
        "    \"\"\"\n",
        "    Unfreeze XLNet weight parameters. They will be updated during training.\n",
        "    \"\"\"\n",
        "    for param in self.xlnet.parameters():\n",
        "      param.requires_grad = True\n",
        "    \n",
        "  def pool_hidden_state(self, last_hidden_state):\n",
        "    \"\"\"\n",
        "    Pool the output vectors into a single mean vector \n",
        "    \"\"\"\n",
        "    last_hidden_state = last_hidden_state[0]\n",
        "    mean_last_hidden_state = torch.mean(last_hidden_state, 1)\n",
        "    return mean_last_hidden_state\n",
        "    \n",
        "model = XLNetForMultiLabelSequenceClassification(num_labels=len(Y_train[0]))\n",
        "#model = torch.nn.DataParallel(model)\n",
        "#model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "762e096d83ad4d2da1a1eabdf1a480a9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=760.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5924d3539110486596cc56ab06983076",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=467042463.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6QuSJxanOMRy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = AdamW(model.parameters(), lr=2e-5, weight_decay=0.01, correct_bias=False)\n",
        "#scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pxCN5nLvOO2a",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "f6e063ff-1328-42ac-b05a-e796faf64ad7"
      },
      "source": [
        "num_epochs=2\n",
        "\n",
        "cwd = os.getcwd()\n",
        "#model_save_path = output_model_file = os.path.join(cwd, \"drive/My Drive/Models/xlnet_toxic.bin\")\n",
        "model_save_path = output_model_file = os.path.join(cwd, 'Models')\n",
        "model, train_loss_set, valid_loss_set = train(model=model,\\\n",
        "                                              num_epochs=num_epochs,\\\n",
        "                                              optimizer=optimizer,\\\n",
        "                                              train_dataloader=train_dataloader,\\\n",
        "                                              valid_dataloader=validation_dataloader,\\\n",
        "                                              model_save_path=model_save_path,\\\n",
        "                                              device=\"cuda\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train loss: 0.012048441357910633\n",
            "Valid loss: 0.010382962599396705\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch:  50%|█████     | 1/2 [00:13<00:13, 13.03s/it]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Saving model at epoch 0 with validation loss of 0.010382962599396705\n",
            "\n",
            "\n",
            "Train loss: 0.011567203793674707\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 100%|██████████| 2/2 [00:24<00:00, 12.33s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Valid loss: 0.01089141210541129\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e5ZPpkIVffCW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        },
        "outputId": "08cfcf05-9007-406c-b68f-f9bd1069d19b"
      },
      "source": [
        "# Plot loss\n",
        "num_epochs = np.arange(len(train_loss_set))\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(10, 5));\n",
        "ax.plot(num_epochs, np.array(train_loss_set), label=\"Train Loss\")\n",
        "ax.plot(num_epochs, np.array(valid_loss_set), 'g-', label=\"Valid Loss\")\n",
        "#ax1.plot(episode_record, lose_record, 'r-', label=\"Lose %\")\n",
        "ax.set_xlabel(\"Number of Epochs\")\n",
        "ax.set_ylabel(\"Loss\")\n",
        "ax.set_title(\"Loss vs Number of Epochs\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Loss vs Number of Epochs')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAFNCAYAAABFbcjcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9d3+/9d7ZkIiJCCyiAICCloBWcMSa61W79bWBTc2UbEiy93a1p9379Zutre2VWur1q0EwYp1A2z96t26VG+p1RqQQFFBQMOiQGVHICCBJO/fH3NShxBgApmcycz1fDzmwZlzzpy5PjMZcuXMmTnm7oiIiIhIeoiEHUBEREREPqNyJiIiIpJGVM5ERERE0ojKmYiIiEgaUTkTERERSSMqZyIiIiJpROVMRKQOZuZm1j2k+z7FzBaa2Q4z+3YYGWozs1Vmdm7YOUSygcqZSJZoyr9czeysoCw9WGv+G2Z2TUixUul7wGx3L3D3e2svNLO/mdluMytPuPxvCDlFJAVUzkSkqdgJXGVmXUPOUS9mFjuMm3UBFh9inevdPT/hcuFh3I+IpCGVM5EsZ2a5ZnaPmf0ruNxjZrnBsrZm9mcz+8TMtpjZ62YWCZZ938zWBm+9LTOzc+rY9hAzW2dm0YR5l5jZO8H0YDMrNbPtZrbezO46SNRPgEeAnx5gHD8zs8cSrncN9rbFgut/M7Ofm9mbNXuazKyNmT0e3P+8Oorf18xshZltMrM7a8YebO9aM1tiZlvN7CUz65KwzM3sm2b2AfDBAfJeZGaLg8f2b2Z2ajD/VeBs4P4g58kHeUzq2u5ZZrbGzH4Y5F5lZmMSlrcys0fNbKOZfWhmP641rvHBuHaY2XtmNiBh8/3M7B0z22ZmM8wsL7jNAX9ORKT+9OIRkR8BQ4F+QF9gMPDjYNl/AWuAdsCxwA8BN7NTgOuBQe5eAHwFWFV7w+4+l/gery8lzL4CeCKY/i3wW3dvCZwEzDxE1l8AlwX3fzhGAVcBHYP7KwF+DxwDLGH/4ncJUAgMAIYB1wKY2TDij8WlxB+b14Ena932YmAI0LN2iKBwPQncENz+eeB/zayZu38p2F7NnrH3D2OcHYC2wTjHAlMSHrP7gFbAicAXgauBrwe5hgM/C+a1BC4CNidsdwRwHtAN6ANcE8yv8+fkMHKLCCpnIgJjgFvcfYO7bwT+h3iBAdgLHAd0cfe97v66x0/IWwXkAj3NLMfdV7n78gNs/0lgNICZFQBf47MisxfobmZt3b3c3eccLKi7rwMmA7cc5lh/7+7L3X0b8AKw3N1fcfdKYBbQv9b6d7j7Fnf/CLinZhzAJOA2d18S3PaXxPcqdUm47W3BbT+tI8dI4C/u/rK77wV+DRwFnF6Psdwb7Kmqudxaa/lP3L3C3V8D/gKMCPZgjgJ+4O473H0V8Bs+e76vA37l7vM8rszdP0y8T3f/l7tvAf6XeKGHA/+ciMhhUDkTkeOBxF/AHwbzAO4EyoC/Bm/v3QTg7mXE9/r8DNhgZk+Z2fHU7Qng0uCt0kuBBQm/8McBJwNLg7cVL0gi7x3AV8ysb9Ij/Mz6hOlP67ieX2v91QnTiY9LF+C3NcUI2AIY8T1Vdd22tn0ec3evDtbveMBb7O/b7n50wuUnCcu2uvvOOrK3BXLY//muud/OwIFKNsC6hOldfPZ41flzIiKHR+VMRP5FvGzUOCGYR7B35b/c/UTib3HdWHNsmbs/4e5nBLd14qVpP+7+HvEC8FX2fUsTd//A3UcD7YPbP21mLQ4W1t03E9+LVXtP0U6gecL1DgfbTpI6J0z/+3EhXqQm1ipHR7n7m4lRD7LdfR5zM7PgvtY2QGaA1rUex5rsm4jv5ar9fNfc72rib/fWy8F+TkSk/lTORLJLjpnlJVxixN9i/LGZtTOztsDNwGMAZnaBmXUPysM24m9nVlv8e7i+FOwN2018r1P1Qe73CeA7wJnE3z4k2P6VZtYu2HP0STD7YNupcRfxtwBPTZi3EDjTzE4ws1bAD5LYzqH8t5m1NrPOQf4ZwfzJwA/MrFcwjlbB8VrJmgmcb2bnmFkO8WO2KoA3D36zevkfM2tmZl8ALgBmuXtVcN+/MLOC4G3YGwmeb2Aq8F0zG2hx3Wu9VVunA/2cNOBYRLKKyplIdnmeeJGqufwM+DlQCrwDvAssCOYB9ABeAcqJHzz/oLvPJn682e3E98SsI77n62Bl6EniB5+/6u6bEuafByw2s3LiHw4YdYBjtPbh7tuBXxE/kL9m3svEy9M7wHzgz4faThKeDba1kPhxW9OC+3qG+J6+p8xsO7CI+J7BpLj7MuBK4gfnbwIuBC509z31yFbzac6ay/yEZeuArcT3lj0OTHL3pcGybxHfy7gCeIN4cX44yDWL+IcungB2AP+PhMf4IA70cyIih8F0zKaISOYws7OAx9y9U9hZROTwaM+ZiIiISBpRORMRERFJI3pbU0RERCSNaM+ZiIiISBpRORMRERFJI7GwAzSUtm3beteuXcOOISIiInJI8+fP3+Tu7epaljHlrGvXrpSWloYdQ0REROSQzOzDAy3T25oiIiIiaUTlTERERCSNqJyJiIiIpBGVMxEREZE0onImIiIikkZUzkRERETSiMqZiIiISBpRORMRERFJIypnIiIiImlE5SxJ7s4j/1jJ5vKKsKOIiIhIBlM5S9KHm3dx2wtLGTllDuu27Q47joiIiGQolbMkdW3bgunXDubjTz5lePGbfLR5V9iRREREJAOltJyZ2XlmtszMyszspjqW55rZjGD5XDPrmrCsj5mVmNliM3vXzPJSmTUZQ09swxPjh7JjdyWXT36T99fvCDuSiIiIZJiUlTMziwIPAF8FegKjzaxnrdXGAVvdvTtwN3BHcNsY8Bgwyd17AWcBe1OVtT76dj6aGROKcGBkcQnvrtkWdiQRERHJIKncczYYKHP3Fe6+B3gKGFZrnWHA9GD6aeAcMzPgy8A77v42gLtvdveqFGatl1M6FPD0pCJa5MYY/dAc5q7YHHYkERERyRCpLGcdgdUJ19cE8+pcx90rgW1AG+BkwM3sJTNbYGbfS2HOw9KlTQtmTSri2Ja5XP3wW8xetiHsSCIiIpIB0vUDATHgDGBM8O8lZnZO7ZXMbIKZlZpZ6caNGxs7I8e1OoqZE4vo3j6fCY+W8pd3Pm70DCIiIpJZUlnO1gKdE653CubVuU5wnFkrYDPxvWx/d/dN7r4LeB4YUPsO3H2Kuxe6e2G7du1SMIRDa5OfyxPjh9K309F868kFzCxdfegbiYiIiBxAKsvZPKCHmXUzs2bAKOC5Wus8B4wNpi8HXnV3B14CTjOz5kFp+yLwXgqzHpFWR+Xw6LjBfL57W7739Ds8/MbKsCOJiIhIE5WychYcQ3Y98aK1BJjp7ovN7BYzuyhYbRrQxszKgBuBm4LbbgXuIl7wFgIL3P0vqcraEJo3izF1bCHn9erALX9+j3v/7wPiPVNEREQkeZYpBaKwsNBLS0vDjkFlVTXf++M7/GnBWsZ/oRs//NqpxD+AKiIiIhJnZvPdvbCuZbHGDpPpYtEIv768LwW5MR56fSXlFZX8/OLTiEZU0EREROTQVM5SIBIxfnZRLwrycrh/dhk7dldy98h+5ETT9cOxIiIiki5UzlLEzPjuV04hPy/G7S8sZdeeKh4cM4C8nGjY0URERCSNaVdOik364kn8/OLezF62gWt+/xblFZVhRxIREZE0pnLWCK4c2oV7RvZj3qqtjHloDlt37gk7koiIiKQplbNGMqxfRyZfOZAl63YwasocNmzfHXYkERERSUMqZ43oP3oey++vGcTqrbsYXlzC6i27wo4kIiIiaUblrJF9vntbHrtuCFt37mFEcQllG8rDjiQiIiJpROUsBANOaM1TE4rYW1XNyOISFq3dFnYkERERSRMqZyHpeXxLZk4sIjcWYfRDc5j/4ZawI4mIiEgaUDkL0Ynt8pn1n6fTNj+XK6e+xesfbAw7koiIiIRM5SxkHY8+ipkTi+jSpjnjHinlxUXrwo4kIiIiIVI5SwPtCnKZMaGIXh1b8s0nFvCnBWvCjiQiIiIhUTlLE62a5/DYuCEM6XYMN858mz+UrAo7koiIiIRA5SyNtMiN8fA1gzj31GP5ybOLeWB2WdiRREREpJGpnKWZvJwov7tyAMP6Hc+dLy3j9heW4u5hxxIREZFGEgs7gOwvJxrh7hH9yM+NMfm15ZRX7OWWi3oTiVjY0URERCTFVM7SVCRi/Pzi3uTnxSh+bQU7K6q48/I+xKLa2SkiIpLJVM7SmJlx03mfo2VeDne+tIzyikruG92fvJxo2NFEREQkRbQbJs2ZGd88uzv/c1EvXn5vPeOmz2NnRWXYsURERCRFVM6aiLGnd+XXw/tSsnwzV02by7ZP94YdSURERFJA5awJuXxgJx4cM4B3125j1JQ5bCqvCDuSiIiINDCVsybmvN7HMW3sIFZuKmfE5BL+9cmnYUcSERGRBqRy1gSdeXI7/jBuCBt3VDB8cgkrN+0MO5KIiIg0EJWzJmpQ12N4csJQPt1bxfDJJSxdtz3sSCIiItIAVM6asN4dWzFz4lBiEWNk8Rz++dHWsCOJiIjIEVI5a+K6ty9g1qQijm6ew5ipc3lz+aawI4mIiMgRUDnLAJ2Pac6siUV0an0U1/x+Hq+8tz7sSCIiInKYVM4yRPuWecyYUMTnOhQw6bH5PLtwbdiRRERE5DConGWQ1i2a8fh1QxjYpTU3zFjIE3M/CjuSiIiI1JPKWYYpyMth+rWDOevkdvzwmXeZ8vflYUcSERGRelA5y0B5OVGKryrk/D7H8cvnl/Kbvy7D3cOOJSIiIkmIhR1AUqNZLMK9o/pTkBvjvlfL2LG7kpsv6EkkYmFHExERkYNQOctg0Yhx26Wn0SI3xrQ3VlJeUcntl55GLKodpiIiIulK5SzDmRk/Pv9UCvJi3PPKB+ysqOSeUf3IjUXDjiYiIiJ10C6ULGBm3HDuyfzkgp68sGgd4x+dz6d7qsKOJSIiInVQOcsi487oxh2XncbrH2zk6ofnsn333rAjiYiISC0qZ1lm5KATuG90f/750Sdc8dActuzcE3YkERERSaByloUu6HM8D11dyAfryxlRXMK6bbvDjiQiIiIBlbMsdfbn2jP92sGs27ab4cVv8tHmXWFHEhEREVJczszsPDNbZmZlZnZTHctzzWxGsHyumXUN5nc1s0/NbGFwmZzKnNlq6IltePy6IezYXcnlk9/k/fU7wo4kIiKS9VJWzswsCjwAfBXoCYw2s561VhsHbHX37sDdwB0Jy5a7e7/gMilVObNd385HM2NCEQAji0t4Z80nIScSERHJbqncczYYKHP3Fe6+B3gKGFZrnWHA9GD6aeAcM9NX2DeyUzoUMGtSES1yY1zx0FzmrtgcdiQREZGslcpy1hFYnXB9TTCvznXcvRLYBrQJlnUzs3+a2Wtm9oUU5hSgS5sWzJpUxLEtc7n64beYvWxD2JFERESyUrp+IOBj4AR37w/cCDxhZi1rr2RmE8ys1MxKN27c2OghM81xrY5i5sQiurfPZ8KjpfzlnY/DjiQiIpJ1UlnO1gKdE653CubVuY6ZxYBWwGZ3r3D3zQDuPh9YDpxc+w7cfYq7F7p7Ybt27VIwhOzTJj+XJycMpV/no/nWkwuYOW/1oW8kIiIiDSaV5Wwe0MPMuplZM2AU8FytdZ4DxgbTlwOvurubWbvgAwWY2YlAD2BFCrNKgpZ5OTx67RA+370t3/vjO0x7Y2XYkURERLJGyspZcAzZ9cBLwBJgprsvNrNbzOyiYLVpQBszKyP+9mXN122cCbxjZguJf1BgkrtvSVVW2d9RzaJMHVvIeb06cOuf3+O3r3yAu4cdS0REJONZpvzCLSws9NLS0rBjZJzKqmq+/8d3+eOCNVx3Rjd+dP6p6AO1IiIiR8bM5rt7YV3LYo0dRpqWWDTCnZf3oSAvxtQ3VlJeUckvLjmNaEQFTUREJBVUzuSQIhHjpxf2JD83xv2zyyivqOTukf3Iiabrh31FRESaLpUzSYqZ8d2vnEJBXozbXljKrj1VPDhmAHk50bCjiYiIZBTt+pB6mfjFk/jFJb2ZvWwDYx9+i/KKyrAjiYiIZBSVM6m3MUO6cM/IfpR+uJUxD81h6849YUcSERHJGCpncliG9etI8ZUDWbJuB6OmzGHD9t1hRxIREckIKmdy2M7teSyPXDOI1Vt3Mby4hNVbdoUdSUREpMlTOZMjcnr3tjx23RC27tzDiOISyjaUhx1JRESkSVM5kyM24ITWzJhYxN4qZ2RxCYvWbgs7koiISJOlciYN4tTjWjJz4lByYxFGPzSH0lU625aIiMjhUDmTBnNiu3xm/efptM3P5appb/H6BxvDjiQiItLkqJxJg+p49FHMnFhElzbNGfdIKS8uWhd2JBERkSZF5UwaXLuCXGZMKKJXx5Z884kF/GnBmrAjiYiINBkqZ5ISrZrn8Ni4IQw98RhunPk2fyhZFXYkERGRJkHlTFKmRW6MaWMHce6px/KTZxfzwOyysCOJiIikPZUzSam8nCi/u3IAF/c7njtfWsbtLyzF3cOOJSIikrZiYQeQzJcTjXDXiH60yI0x+bXllFfs5ZaLehOJWNjRRERE0o7KmTSKSMT4+cW9yc+LUfzaCsp3V3Ln8L7kRLXzVkREJJHKmTQaM+MHXz2Vlnk53PnSMnbuqeK+0f3Jy4mGHU1ERCRtaLeFNLpvnt2dW4b14uX31jNu+jx2VlSGHUlERCRtqJxJKK4u6spvhvelZPlmrpo2l2279oYdSUREJC2onEloLhvYiQfHDGTR2u2MemgOG3dUhB1JREQkdCpnEqrzendg6thCVm4qZ2RxCWs/+TTsSCIiIqFSOZPQnXlyOx4bN4SNOyoYMbmElZt2hh1JREQkNCpnkhYKux7DkxOG8uneKoZPLmHJx9vDjiQiIhIKlTNJG707tmLmxCJiEWNkcQn//Ghr2JFEREQancqZpJXu7fOZNamI1i2aMWbqXN5cvinsSCIiIo1K5UzSTudjmjNrYhGdWh/FNb+fxyvvrQ87koiISKNROZO01L5lHjMmFHFqhwImPjafZxeuDTuSiIhIo1A5k7TVukUzHh8/lMIurblhxkKemPtR2JFERERSTuVM0lp+bozp1w7m7FPa88Nn3qX4teVhRxIREUkplTNJe3k5USZfOZAL+hzHbS8s5Td/XYa7hx1LREQkJWJhBxBJRrNYhN+O6k9+boz7Xi1jx+5Kbr6gJ5GIhR1NRESkQamcSZMRjRi3XXoa+bkxpr6xkvKKSm6/9DRiUe0AFhGRzKFyJk2KmfGj80+lIC+Hu195n50Vldwzqh+5sWjY0URERBqEdjlIk2NmfOfcHvzkgp68sGgd4x+dz6d7qsKOJSIi0iBUzqTJGndGN351WR/e+GAjVz88l+2794YdSURE5IipnEmTNmJQZ+4bPYCFqz/hiofmsLm8IuxIIiIiR0TlTJq88/scx5SrC/lgfTkjp8xh3bbdYUcSERE5bCpnkhHOPqU9068dzLptuxle/CYfbd4VdiQREZHDktJyZmbnmdkyMyszs5vqWJ5rZjOC5XPNrGut5SeYWbmZfTeVOSUzDD2xDU+MH8KO3ZVcPvlN3l+/I+xIIiIi9ZaycmZmUeAB4KtAT2C0mfWstdo4YKu7dwfuBu6otfwu4IVUZZTM06fT0cycWATAiOIS3lnzSciJRERE6ieVe84GA2XuvsLd9wBPAcNqrTMMmB5MPw2cY2YGYGYXAyuBxSnMKBno5GMLmDWpiPzcGFc8NJe5KzaHHUlERCRpqSxnHYHVCdfXBPPqXMfdK4FtQBszywe+D/zPwe7AzCaYWamZlW7cuLHBgkvT16VNC56edDodWuVx9cNvMXvphrAjiYiIJCVdPxDwM+Budy8/2EruPsXdC929sF27do2TTJqMDq3ymDFhKN3b5zP+0VL+8s7HYUcSERE5pFSWs7VA54TrnYJ5da5jZjGgFbAZGAL8ysxWATcAPzSz61OYVTJUm/xcnpwwlP4nHM23nlzAzHmrD30jERGREKWynM0DephZNzNrBowCnqu1znPA2GD6cuBVj/uCu3d1967APcAv3f3+FGaVDNYyL4dHrx3CGT3a8b0/vsO0N1aGHUlEROSAUlbOgmPIrgdeApYAM919sZndYmYXBatNI36MWRlwI7Df122INISjmkV56OqBfLV3B27983v89pUPcPewY4mIiOzHMuUXVGFhoZeWloYdQ9JcZVU1N/3pXZ6ev4brzujGj84/leADwiIiIo3GzOa7e2Fdy2KNHUYkTLFohF9d1of83BhT31hJeUUlv7jkNKIRFTQREUkPKmeSdSIR46cX9qQgL8Z9r5ZRXlHJXSP60SyWrh9eFhGRbKJyJlnJzPivL59Cfm6M215Yyq49VTw4ZgB5OdGwo4mISJbTrgLJahO/eBK/uKQ3s5dtYOzDb7Fj996wI4mISJZTOZOsN2ZIF+4Z2Y/SD7dy5dS5bN25J+xIIiKSxVTORIBh/TpSfOVAlqzbwagpc9iwfXfYkUREJEslVc7MrIWZRYLpk83sIjPLSW00kcZ1bs9jeeSaQazeuovhxSWs3rIr7EgiIpKFkt1z9ncgz8w6An8FrgIeSVUokbCc3r0tj183hE927WX45BLKNhz09K4iIiINLtlyZu6+C7gUeNDdhwO9UhdLJDz9T2jNUxOGUlntjCwuYdHabWFHEhGRLJJ0OTOzImAM8Jdgnr5zQDLWqce1ZNakIvJyoox+aA6lq7aEHUlERLJEsuXsBuAHwDPB+TFPBGanLpZI+Lq1bcHMSUW0y8/lqmlv8ff3N4YdSUREskBS5czdX3P3i9z9juCDAZvc/dspziYSuo5HH8WMiUV0bduC66aX8uKidWFHEhGRDJfspzWfMLOWZtYCWAS8Z2b/ndpoIumhXUEuT40fSq+OLfnmEwv44/w1YUcSEZEMluzbmj3dfTtwMfAC0I34JzZFskKr5jk8Nm4IQ088hv+a9TaPlqwKO5KIiGSoZMtZTvC9ZhcDz7n7XsBTF0sk/bTIjTFt7CD+o+ex3PzsYh6YXRZ2JBERyUDJlrNiYBXQAvi7mXUBtqcqlEi6ysuJ8uCYAVzc73jufGkZt7+wFHf9nSIiIg0nlsxK7n4vcG/CrA/N7OzURBJJbznRCHeN6EeL3BiTX1vOjt17uXVYbyIRCzuaiIhkgKTKmZm1An4KnBnMeg24BdC3c0pWikSMn1/cm4K8HCa/tpydFZXcObwvOVGdrlZERI5MUuUMeJj4pzRHBNevAn5P/IwBIlnJzLjpq5+jIC/GnS8tY+eeKu4b3Z+8HH0/s4iIHL5k/8w/yd1/6u4rgsv/ACemMphIU/HNs7tzy7BevPzeesZNn8fOisqwI4mISBOWbDn71MzOqLliZp8HPk1NJJGm5+qirtw1oi9zVmzhymlz2bZrb9iRRESkiUq2nE0CHjCzVWa2CrgfmJiyVCJN0KUDOvHAFQNYvHY7ox6aw8YdFWFHEhGRJijZ0ze97e59gT5AH3fvD3wppclEmqDzendg2jWFrNq0k5HFJaz9RDuYRUSkfur10TJ33x6cKQDgxhTkEWnyvtCjHX8YN5iN5RUM/92brNy0M+xIIiLShBzJ5/71pU4iB1DY9RieHD+U3ZXVDJ9cwpKP9Z3NIiKSnCMpZ/padJGD6N2xFTMnFhGLGCOLS1jw0dawI4mISBNw0HJmZjvMbHsdlx3A8Y2UUaTJ6t4+n1mTimjdohlXTp3Lm2Wbwo4kIiJp7qDlzN0L3L1lHZcCd0/2C2xFslrnY5oza2IRnVs355pH5vHKe+vDjiQiImlM55oRaQTtW+bx1IShnNqhgImPzefZhWvDjiQiImlK5UykkbRu0YzHxw+lsEtrbpixkCfmfhR2JBERSUMqZyKNKD83xvRrB3P2Ke354TPvUvza8rAjiYhImlE5E2lkeTlRiq8ayAV9juO2F5by65eW4a4PP4uISJwO6hcJQU40wm9H9Sc/N8b9s8sor6jk5gt6Eono6wNFRLKdyplISKIR47ZLT6MgL8ZDr69kx+5K7rjsNGJR7dAWEclmKmciITIzfvi1UynIy+Gul99n155K7hnVj9xYNOxoIiISEv2JLhIyM+Pb5/Tg5gt68sKidVw3vZRdeyrDjiUiIiFRORNJE9ee0Y1fXdaHf5Rt4uppb7F9996wI4mISAhUzkTSyIhBnblv9ADeXvMJVzw0h83lFWFHEhGRRqZyJpJmzu9zHFOuLuSD9eWMKC5h3bbdYUcSEZFGlNJyZmbnmdkyMyszs5vqWJ5rZjOC5XPNrGswf7CZLQwub5vZJanMKZJuzj6lPY9eO5j12yu4fPKbfLh5Z9iRRESkkaSsnJlZFHgA+CrQExhtZj1rrTYO2Oru3YG7gTuC+YuAQnfvB5wHFJuZPlkqWWXIiW14YvwQdlZUMnxyCe+v3xF2JBERaQSp3HM2GChz9xXuvgd4ChhWa51hwPRg+mngHDMzd9/l7jUfV8sD9PXpkpX6dDqaGROLABhRXMLbqz8JOZGIiKRaKstZR2B1wvU1wbw61wnK2DagDYCZDTGzxcC7wKSEsiaSVU4+toCnJ51OQV6MMVPnMnfF5rAjiYhICqXtBwLcfa679wIGAT8ws7za65jZBDMrNbPSjRs3Nn5IkUZyQpvmzJp4Oh1a5XH1w28xe+mGsCOJiEiKpLKcrQU6J1zvFMyrc53gmLJWwD67Bdx9CVAO9K59B+4+xd0L3b2wXbt2DRhdJP10aJXHjAlD6XFsPuMfLeXP7/wr7EgiIpICqSxn84AeZtbNzJoBo4Dnaq3zHDA2mL4ceNXdPbhNDMDMugCfA1alMKtIk9AmP5cnxg+l/wlH8+0n/8nMeasPfSMREWlSUlbOgmPErgdeApYAM919sZndYmYXBatNA9qYWRlwI1DzdRtnAG+b2ULgGeAb7r4pVVlFmpKWeTk8eu0QvtCjHd/74ztMe2Nl2JFERKQBmXtmfBCysLDQS0tLw44h0mj2VFbznaf+yW+IL3QAABocSURBVAuL1nHDuT34zjk9MLOwY4mISBLMbL67F9a1LG0/ECAiB9csFuG+0f25fGAn7nnlA37xlyVkyh9bIiLZTF/sKtKExaIRfnVZH/JzY0x9YyXlFZX84pLTiEa0B01EpKlSORNp4iIR46cX9qRlXox7Xy2jvKKSu0b0o1lMO8ZFRJoilTORDGBm3PjlU8jPi/HL55eys6KS3105kLycaNjRRESknvSntUgGmXDmSfzyktP42/sbGfvwW+zYvTfsSCIiUk8qZyIZ5oohJ3DPyH7M/3ArV06dy9ade8KOJCIi9aByJpKBhvXryOQrB7Jk3Q5GTilhw/bdYUcSEZEkqZyJZKhzex7LI18fxNqtnzK8uITVW3aFHUlERJKgciaSwU4/qS2PXTeET3btZfjkEso2lIcdSUREDkHlTCTD9T+hNU9NGEpltTOiuIRFa7eFHUlERA5C5UwkC5x6XEtmTSriqJwoo6fMoXTVlrAjiYjIAaiciWSJbm1bMGtSEe0Kcrlq2lv8/f2NYUcSEZE6qJyJZJHjjz6KGROL6Nq2BddNL+XFRR+HHUlERGpRORPJMu0Kcnlq/FB6d2zJNx5fwB/nrwk7koiIJFA5E8lCrZrn8IdxQyg6qQ3/NettHi1ZFXYkEREJqJyJZKkWuTGmjR3Ef/Q8lpufXcwDs8tw97BjiYhkPZUzkSyWlxPlwTEDuKR/R+58aRm3v7hUBU1EJGSxsAOISLhyohF+M7wvLXKjFL+2gvLdldw6rDeRiIUdTUQkK6mciQiRiHHrsN4U5OXwu78tp7yikl8P70tOVDvXRUQam8qZiABgZnz/vM9RkBfjVy8uY2dFFfdf0Z+8nGjY0UREsor+LBaRfXzjrO7cOqwXryxZz7jp89hZURl2JBGRrKJyJiL7uaqoK3eN6MucFVu4ctpctu3aG3YkEZGsoXImInW6dEAnHrhiAIvXbmfklBI27qgIO5KISFZQORORAzqvdwemXVPIh5t3MbK4hLWffBp2JBGRjKdyJiIH9YUe7XjsusFsLK9g+O/eZMXG8rAjiYhkNJUzETmkgV2O4cnxQ6morGZEcQlLPt4ediQRkYylciYiSendsRUzJhaRE40wsriEBR9tDTuSiEhGUjkTkaR1b5/PrElFtG7RjCunzuXNsk1hRxIRyTgqZyJSL51aN2fWxCI6t27ONY/M4+X31ocdSUQko6iciUi9tW+Zx4yJQzn1uJZMemw+zy5cG3YkEZGMoXImIofl6ObNePy6IQzq2pobZizk8bkfhh1JRCQjqJyJyGHLz43xyNcHc/Yp7fnRM4uY/NrysCOJiDR5KmcickTycqIUXzWQC/sez+0vLOXXLy3D3cOOJSLSZMXCDiAiTV9ONMI9I/uRnxvl/tlllFdUcvMFPYlELOxoIiJNjsqZiDSIaMT45SWnkZ8b46HXV7JjdyV3XHYasah20IuI1IfKmYg0GDPjh187lYK8HO56+X12VlTy29H9yI1Fw44mItJk6E9aEWlQZsa3z+nBzRf05MXF67hueim79lSGHUtEpMlQORORlLj2jG786vI+/KNsE1dPe4vtu/eGHUlEpElQORORlBlR2Jn7Rg/g7TWfMHrKHDaXV4QdSUQk7amciUhKnd/nOB66upDlG8sZUVzCum27w44kIpLWUlrOzOw8M1tmZmVmdlMdy3PNbEawfK6ZdQ3m/4eZzTezd4N/v5TKnCKSWmed0p5Hrx3C+u0VXD75TT7cvDPsSCIiaStl5czMosADwFeBnsBoM+tZa7VxwFZ37w7cDdwRzN8EXOjupwFjgT+kKqeINI7B3Y7hifFD2FlRyfDJJby/fkfYkURE0lIq95wNBsrcfYW77wGeAobVWmcYMD2Yfho4x8zM3f/p7v8K5i8GjjKz3BRmFZFG0KfT0cycWATAiOIS3l79SciJRETSTyrLWUdgdcL1NcG8Otdx90pgG9Cm1jqXAQvcXUcSi2SAHscW8PSk0ynIizFm6lzmrNgcdiQRkbSS1h8IMLNexN/qnHiA5RPMrNTMSjdu3Ni44UTksJ3QpjmzJp5Oh1Z5jH34LWYv3RB2JBGRtJHKcrYW6JxwvVMwr851zCwGtAI2B9c7Ac8AV7v78rruwN2nuHuhuxe2a9eugeOLSCp1aJXHzIlFnHxsAeMfLeXP7/zr0DcSEckCqSxn84AeZtbNzJoBo4Dnaq3zHPED/gEuB151dzezo4G/ADe5+z9SmFFEQnRMi2Y8Pn4I/U84mm8/+U9mzPso7EgiIqFLWTkLjiG7HngJWALMdPfFZnaLmV0UrDYNaGNmZcCNQM3XbVwPdAduNrOFwaV9qrKKSHha5uXw6LVD+EKPdnz/j+8y9fUVYUcSEQmVuXvYGRpEYWGhl5aWhh1DRA7TnspqbpjxT55/dx03nNuD75zTAzMLO5aISEqY2Xx3L6xrWayxw4iI1KVZLMK9o/rTotm73PPKB+zYXcmPzz9VBU1Eso7KmYikjVg0wh2X9aFFboxpb6ykfHclv7z0NKIRFTQRyR4qZyKSViIR46cX9qRlXox7Xy2jfE8ld4/oR7NYWn/zj4hIg1E5E5G0Y2bc+OVTKMjL4RfPL2FXRSW/u3IgeTnRsKOJiKSc/hQVkbQ1/swT+eUlp/G39zcy9uG32LF7b9iRRERSTuVMRNLaFUNO4J6R/Zj/4VbGTJ3L1p17wo4kIpJSKmcikvaG9etI8VUDWbpuByOnlLBh++6wI4mIpIzKmYg0CeeceiyPfH0Qa7d+yuWTS1i9ZVfYkUREUkLlTESajNNPastj1w1h26d7GT65hLIN5WFHEhFpcCpnItKk9D+hNTMmDqWy2hlRXMKitdvCjiQi0qBUzkSkyflch5bMmlTEUTlRRk+Zw7xVW8KOJCLSYFTORKRJ6ta2BbMmFdGuIJerps3l7+9vDDuSiEiDUDkTkSbr+KOPYuakIk5sm89100t5cdHHYUcSETliKmci0qS1zc/lyQlD6d2xJd94fAFPz18TdiQRkSOiciYiTV6ro3L4w7ghnH5SW747622mv7kq7EgiIodN5UxEMkKL3BhTxxby5Z7H8tPnFvPA7DLcPexYIiL1pnImIhkjLyfKg2MGcEn/jtz50jJuf3GpCpqINDmxsAOIiDSkWDTCb4b3pUVulOLXVrBjdyW3DutNNGJhRxMRSYrKmYhknEjEuHVYbwrycvjd35azs6KSXw/vS05UbxaISPpTORORjGRmfP+8z1GQF+NXLy5jZ0UV91/Rn7ycaNjRREQOSn9GikhG+8ZZ3bl1WC9eWbKeax+Zx86KyrAjiYgclMqZiGS8q4q6cvfIvsxduYUrp81l2669YUcSETkglTMRyQqX9O/Eg2MGsHjtdkZOKWHjjoqwI4mI1EnlTESyxld6deDhawbx4eZdjCguYe0nn4YdSURkPypnIpJVzujRlseuG8ym8gqG/+5NVmwsDzuSiMg+VM5EJOsM7HIMT00YSkVlNSOKS1jy8fawI4mI/JvKmYhkpV7Ht2LmpCJyohFGFpew4KOtYUcSEQFUzkQki53ULp9Zk4o4pkUzrpw6l3+UbQo7koiIypmIZLdOrZszc1IRnVs35+uPzOPl99aHHUlEspzKmYhkvfYFecyYOJRTj2vJpMfm8+zCtWFHEpEspnImIgIc3bwZj183hEFdW3PDjIU8PvfDsCOJSJZSORMRCeTnxnjk64P50int+dEzi5j82vKwI4lIFtKJz0VEEuTlRJl81UBunPk2t7+wlDVbd9Hr+FZEzTCDiBnRSHw6GjEiZkSC+REzIpGE6YTr8XXjJ2SPBsv220Yw/e/7itSsG9wusv/0Z/dFkMvCfghF5AipnImI1JITjXDPyH4U5MV4bM5HYcept8SiFjHqLoO1y11NidyvhBrRYFm8WB6giB6gLNa+X0vIk3j7msK733Qd5TRSVyneZ8zxzInT+xTm2ttImI7WjPNgRThSz3HWKumJY7PgPkQSmbuHnaFBFBYWemlpadgxRCTDbC6vYG+VU+VOdbVT7U61E/+3+rPpqmrHa6bdcXeqqoP13KlOnN7vOlQF2/ba0wnr1Nz//vcVX1YVrOfBOnXlrLlUVcfXS5yuuf1nt9n3fvfZRjUJ40y43zrGlpghnj2Y9oTpOsafePsM+VVVJztAgd53z+pnBfFgRfZAe2MPVC7326MbSdy7y/7FO7JviU0sv/sX+TrKb+IfDXX8AVHXfSUz/mid69cea3J7riMGzWIRmjdL7f4rM5vv7oV1LdOeMxGRg2iTnxt2BIF/l7g6y2l1YomtVQ5rl+IDFsG6i+w+91W7hNYuwgnb+nfxrHaqfN8ifKDCvN826iiyBy3dSRT+qmpnT1WShf8A4//s/g5+/03ZWae045GvDw7t/lXOREQk7dW8pRrFyImGnUYOxWvt+d2vCNexN/agRbiO4ul1Tf/7/j4rwvvtxa6157eu7XVsfVSoj5/KWZJ27tnJFX+6gqhFiUViRCPBv/bZv/vNi0QPvP4BljXmNiIW0bEOIiLS4Kzm2EX0O+ZwqJwlqcqr+PCTD6nyKiqrK6mqrtpnurK6kiqv2mc6cb10VZ8yV9d69S2E6VxWVWRFJFNUezXVXk1VdVX8X6866PVk1mky1xtgO0WdirjrK3eF9vypnCWpZW5LFk5aeFi3je8qra5Xmau93uEUwgbbRpLr76nak9R62Vpk610qU1WQU7wNFdnsUfN/W1r9Ym6CRSAV1zNNxCJELPLv/2OikWiDX49GojSzZkQsQvOc5qGON6XlzMzOA34LRIGp7n57reW5wKPAQGAzMNLdV5lZG+BpYBDwiLtfn8qcqRY/ViJKlCjNos3CjpPW6iqyaVdWkyyyB9tuskX2UNnSVe0Cl/I9oCksq1GLhvsLlyPfTqqyOU38qO9aDEvZL/7E682izYjE6nl7jvx+G2NsqbhuZN/396WsnJlZFHgA+A9gDTDPzJ5z9/cSVhsHbHX37mY2CrgDGAnsBn4C9A4ukiVUZJOXbJFNVVk9klJ5uEX2SHKnk8b45RaLxGhmzQ7vF3Qa/WJuzOvZVgAkfaVyz9lgoMzdVwCY2VPAMCCxnA0DfhZMPw3cb2bm7juBN8ysewrziTRpKrLJSyyy9S2a1V7doEUgYjprnogcXCrLWUdgdcL1NcCQA63j7pVmtg1oA2xK5g7MbAIwAeCEE0440rwikqESiyz6GgYRSXNN+k84d5/i7oXuXtiuXbuw44iIiIgcsVSWs7VA54TrnYJ5da5jZjGgFfEPBoiIiIhkpVSWs3lADzPrZmbNgFHAc7XWeQ4YG0xfDrzqmXKyTxEREZHDkLJjzoJjyK4HXiJ+lMfD7r7YzG4BSt39OWAa8AczKwO2EC9wAJjZKqAl0MzMLga+XOuTniIiIiIZJ6Xfc+buzwPP15p3c8L0bmD4AW7bNZXZRERERNJRk/5AgIiIiEimUTkTERERSSMqZyIiIiJpROVMREREJI2onImIiIikEcuUrxUzs43Ah41wV21J8vRSGSibxw7ZPX6NPXtl8/izeeyQ3eNvjLF3cfc6T2+UMeWssZhZqbsXhp0jDNk8dsju8Wvs2Tl2yO7xZ/PYIbvHH/bY9bamiIiISBpRORMRERFJIypn9Tcl7AAhyuaxQ3aPX2PPXtk8/mweO2T3+EMdu445ExEREUkj2nMmIiIikkZUzgJmdp6ZLTOzMjO7qY7luWY2I1g+18y6Jiz7QTB/mZl9pTFzN5Qkxn+jmb1nZu+Y2f+ZWZeEZVVmtjC4PNe4yY9cEmO/xsw2JozxuoRlY83sg+AytnGTH7kkxn53wrjfN7NPEpY19ef9YTPbYGaLDrDczOze4LF5x8wGJCxr0s87JDX+McG43zWzN82sb8KyVcH8hWZW2nipG0YSYz/LzLYl/HzfnLDsoK+ZdJfE2P87YdyLgtf5McGyJv28A5hZZzObHfw+W2xm36ljnfBf++6e9RcgCiwHTgSaAW8DPWut8w1gcjA9CpgRTPcM1s8FugXbiYY9phSM/2ygeTD9nzXjD66Xhz2GFI/9GuD+Om57DLAi+Ld1MN067DE15Nhrrf8t4OFMeN6D/GcCA4BFB1j+NeAFwIChwNxMeN7rMf7Ta8YFfLVm/MH1VUDbsMeQwrGfBfy5jvn1es2k4+VQY6+17oXAq5nyvAdjOA4YEEwXAO/X8X9+6K997TmLGwyUufsKd98DPAUMq7XOMGB6MP00cI6ZWTD/KXevcPeVQFmwvabkkON399nuviu4Ogfo1MgZUyWZ5/5AvgK87O5b3H0r8DJwXopypkJ9xz4aeLJRkjUCd/87sOUgqwwDHvW4OcDRZnYcTf95Bw49fnd/MxgfZNZrPpnn/kCO5P+LtFDPsWfUax7A3T929wXB9A5gCdCx1mqhv/ZVzuI6AqsTrq9h/yfr3+u4eyWwDWiT5G3TXX3HMI74XxU18sys1MzmmNnFqQiYQsmO/bJg9/bTZta5nrdNV0nnD97G7ga8mjC7KT/vyTjQ49PUn/fDUfs178BfzWy+mU0IKVOqFZnZ22b2gpn1CuZlzXNvZs2JF48/JszOqOfd4ocn9Qfm1loU+ms/loqNSuYysyuBQuCLCbO7uPtaMzsReNXM3nX35eEkTIn/BZ509wozm0h8D+qXQs7U2EYBT7t7VcK8TH/eBTCzs4mXszMSZp8RPPftgZfNbGmwRyZTLCD+811uZl8D/h/QI+RMje1C4B/unriXLWOedzPLJ148b3D37WHnqU17zuLWAp0TrncK5tW5jpnFgFbA5iRvm+6SGoOZnQv8CLjI3Stq5rv72uDfFcDfiP8l0lQccuzuvjlhvFOBgcneNs3VJ/8oar290cSf92Qc6PFp6s970sysD/Gf+WHuvrlmfsJzvwF4hqZ3KMdBuft2dy8Ppp8HcsysLVn03HPw13yTft7NLId4MXvc3f9Uxyrhv/bDOCAv3S7E9yCuIP62Tc1Bnr1qrfNN9v1AwMxguhf7fiBgBU3vAwHJjL8/8QNhe9Sa3xrIDabbAh/QhA6QTXLsxyVMXwLMCaaPAVYGj0HrYPqYsMfUkGMP1vsc8QOBLVOe94RxdOXAB4Wfz74HBb+VCc97PcZ/AvFjaE+vNb8FUJAw/SZwXthjaeCxd6j5eSdeQD4Kfg6Ses2k++VgYw+WtyJ+XFqLDHzeDXgUuOcg64T+2tfbmsSPITOz64GXiH8a52F3X2xmtwCl7v4cMA34g5mVEf+hHRXcdrGZzQTeAyqBb/q+b/2kvSTHfyeQD8yKfw6Cj9z9IuBUoNjMqonvib3d3d8LZSCHIcmxf9vMLiL+/G4h/ulN3H2Lmd0KzAs2d4vv+xZAWkty7BD/WX/Kg/+dAk36eQcwsyeJfyqvrZmtAX4K5AC4+2TgeeKf2ioDdgFfD5Y16ee9RhLjv5n4cbUPBq/5So+fCPpY4JlgXgx4wt1fbPQBHIEkxn458J9mVgl8CowKfv7rfM2EMITDlsTYIf5H6F/dfWfCTZv88x74PHAV8K6ZLQzm/ZD4HyNp89rXGQJERERE0oiOORMRERFJIypnIiIiImlE5UxEREQkjaiciYiIiKQRlTMRERGRNKJyJiKNyszczH6TcP27ZvazBtr2I2Z2eUNs6xD3M9zMlpjZ7Frzu5rZp2a2MOFydQPe71lm9ueG2p6IpCd9z5mINLYK4FIzu83dN4UdpoaZxTx+3txkjAPGu/sbdSxb7u79GjCaiGQZ7TkTkcZWCUwB/r/aC2rv+TKz8uDfs8zsNTN71sxWmNntZjbGzN4ys3fN7KSEzZwbnJD9fTO7ILh91MzuNLN5wQnsJyZs93Uze474F0nXzjM62P4iM7sjmHcz8fNMTjOzO5MdtJmVm9ndZrbYzP7PzNoF8/sFJ49/x8yeMbPWwfzuZvZKcPLtBQljzDezp81sqZk9bsG3ggaPyXvBdn6dbC4RST8qZyIShgeAMWbWqh636QtMIn52gquAk919MPFzP34rYb2uxE+5cz4w2czyiO/p2ubug4BBwHgz6xasPwD4jrufnHhnZnY8cAfxk9z3AwaZ2cXufgtQCoxx9/+uI+dJtd7W/EIwvwXxMy/0Al4j/s3sED+VzPfdvQ/wbsL8x4EH3L0vcDrwcTC/P3AD0BM4Efi8mbUh/q3uvYLt/PxQD6aIpC+VMxFpdO6+nXgp+XY9bjbP3T/2+EnolwN/Dea/S7yQ1Zjp7tXu/gHx8yB+DvgycHVwupa5xE9L1CNY/y13X1nH/Q0C/ubuG4O3Ox8Hzkwi53J375dweT2YXw3MCKYfA84IyunR7v5aMH86cKaZFQAd3f0ZAHff7e67EvKucfdqYGEw9m3AbuJ78y4lfsoZEWmiVM5EJCz3EN+j1SJhXiXB/0tmFiF+cukaFQnT1QnXq9n3+Nna56Rz4icw/lZCYerm7jXlbifhONxz5yU+DlVAzbFyg4GngQuApnjOQxEJqJyJSCiCEwbPJF7QaqwCBgbTFxGckLmehptZJDhG60RgGfETVf+nmeUAmNnJZtbiYBsB3gK+aGZtzSwKjCb+duThihA/oTbAFcAb7r4N2Jrw1udVwGvuvgNYY2YXB3lzzaz5gTZsZvlAK3d/nvixfH2PIKeIhEyf1hSRMP0GuD7h+kPAs2b2NvG9P4ezV+sj4sWqJTDJ3Xeb2VTib/8tCA6g3whcfLCNuPvHZnYTMJv4nre/uPuzSdz/ScHbpzUedvd7iY9lsJn9GNgAjAyWjyV+bFxz4m/Dfj2YfxVQbGa3AHuB4Qe5zwLij1tekPXGJHKKSJoy98Pdsy4iIskys3J3zw87h4ikP72tKSIiIpJGtOdMREREJI1oz5mIiIhIGlE5ExEREUkjKmciIiIiaUTlTERERCSNqJyJiIiIpBGVMxEREZE08v8DH1LKu46VFMEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sHt9MdUoOaLl",
        "colab_type": "text"
      },
      "source": [
        "## Train Model From Previous Checkpoint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lzZEnoI3Ob2u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cwd = os.getcwd()\n",
        "model_save_path = output_model_file = os.path.join(cwd, \"Models\")\n",
        "model, start_epoch, lowest_eval_loss, train_loss_hist, valid_loss_hist = load_model(model_save_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HrTdby8Uaqqb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = AdamW(model.parameters(), lr=2e-5, weight_decay=0.01, correct_bias=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiwYWrPsREQ-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 336
        },
        "outputId": "8c5edb77-2e02-4fe4-b562-0ee10098e682"
      },
      "source": [
        "num_epochs=3\n",
        "model, train_loss_set, valid_loss_set = train(model=model,\\\n",
        "                                              num_epochs=num_epochs,\\\n",
        "                                              optimizer=optimizer,\\\n",
        "                                              train_dataloader=train_dataloader,\\\n",
        "                                              valid_dataloader=validation_dataloader,\\\n",
        "                                              model_save_path=model_save_path,\\\n",
        "                                              train_loss_set=train_loss_hist,\\\n",
        "                                              valid_loss_set=valid_loss_hist,\\\n",
        "                                              lowest_eval_loss=lowest_eval_loss,\\\n",
        "                                              start_epoch=start_epoch,\\\n",
        "                                              device=\"cuda\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train loss: 0.017967663146555422\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch:  33%|███▎      | 1/3 [00:11<00:23, 11.65s/it]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Valid loss: 0.012439918518066407\n",
            "\n",
            "\n",
            "Train loss: 0.010380762442946435\n",
            "Valid loss: 0.00782701950520277\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch:  67%|██████▋   | 2/3 [00:24<00:12, 12.12s/it]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Saving model at epoch 1 with validation loss of 0.00782701950520277\n",
            "\n",
            "\n",
            "Train loss: 0.00686686160042882\n",
            "Valid loss: 0.0056544291786849495\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 100%|██████████| 3/3 [00:38<00:00, 12.68s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Saving model at epoch 2 with validation loss of 0.0056544291786849495\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cew4rYj16aJl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        },
        "outputId": "f08b8e89-b918-4667-e90e-6423f88cdd30"
      },
      "source": [
        "# Plot loss\n",
        "num_epochs = np.arange(len(train_loss_set))\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(10, 5));\n",
        "ax.plot(num_epochs, np.array(train_loss_set), label=\"Train Loss\")\n",
        "ax.plot(num_epochs, np.array(valid_loss_set), 'g-', label=\"Valid Loss\")\n",
        "#ax1.plot(episode_record, lose_record, 'r-', label=\"Lose %\")\n",
        "ax.set_xlabel(\"Number of Epochs\")\n",
        "ax.set_ylabel(\"Loss\")\n",
        "ax.set_title(\"Loss vs Number of Epochs\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Loss vs Number of Epochs')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAFNCAYAAABFbcjcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxcV33//9dH+2JZu1fJtmx5k4mzOSYbSQgBnJ21hJ2SFtI2tDQNhUAhidkhFCjQBgoUUkgJ5PuDxk5I0pB9IbGT2E4seZF3OV60W7Z26fP7Y66UkSzZsq3RHY3ez8djHpq5987M52iU8TvnnnuOuTsiIiIiEh+Swi5ARERERF6ncCYiIiISRxTOREREROKIwpmIiIhIHFE4ExEREYkjCmciIiIicUThTERkCGbmZlYe0nsvNLN1ZtZiZn8fRg2DmdlOM7ss7DpEJgKFM5EJYjz/42pmlwRh6d8HbX/azD4WUlmx9M/AY+6e4+7/NninmT1uZu1mdjjqtiqEOkUkBhTORGS8OAJ82MzmhFzHCTGzlJN42mxg43GOudHdJ0Xdrj6J9xGROKRwJjLBmVm6mX3PzF4Lbt8zs/RgX5GZrTazJjNrMLOnzCwp2PdZM9sbnHrbbGZvGeK132hm+80sOWrbO81sQ3B/uZmtNbNDZnbAzP71GKU2Ab8Abh2mHbeZ2a+iHs8JettSgsePm9lXzOzZvp4mMys0s18H779miOB3hZltN7M6M/t2X9uD1/u4mVWZWaOZPWRms6P2uZn9nZltBbYOU+81ZrYx+N0+bmaLg+2PAm8GfhjUueAYv5OhXvcSM6sxs88Hde80sw9G7c81s7vMrNbMdpnZvwxq118H7Woxs0ozOyvq5c8wsw1m1mxm95hZRvCcYf9OROTE6T8eEfkCcC5wBnA6sBz4l2DfPwE1QDEwFfg84Ga2ELgROMfdc4C3AzsHv7C7P0+kx+vSqM0fAO4O7n8f+L67TwbmAb89Tq1fBd4dvP/JuA74MDAzeL/ngP8CCoAqjg5+7wSWAWcB1wIfBzCza4n8Lt5F5HfzFPA/g577DuCNQMXgIoLA9T/Ap4PnPwCsMrM0d780eL2+nrEtJ9HOaUBR0M6PAj+J+p39AMgF5gIXAx8B/jKo673AbcG2ycA1QH3U6/4FsAIoA5YCHwu2D/l3chJ1iwgKZyICHwRWuvtBd68FbicSYAC6gOnAbHfvcvenPLIgbw+QDlSYWaq773T3bcO8/v8A7wcwsxzgCl4PMl1AuZkVufthd//zsQp19/3AncDKk2zrf7n7NndvBv4IbHP3R9y9G/gdcOag47/p7g3uvhv4Xl87gBuAr7t7VfDcrxHpVZod9dyvB89tG6KO9wH3u/v/uXsXcAeQCZx/Am35t6Cnqu/25UH7v+juHe7+BHA/8BdBD+Z1wC3u3uLuO4Hv8Prn/VfAt9x9jUdUu/uu6Pd099fcvQFYRSTQw/B/JyJyEhTORGQGEP0P8K5gG8C3gWrg4eD03ucA3L2aSK/PbcBBM/uNmc1gaHcD7wpOlb4LeCnqH/zrgQXApuC04lUjqPebwNvN7PQRt/B1B6Lutw3xeNKg4/dE3Y/+vcwGvt8XjIAGwIj0VA313MEG/M7dvTc4fuawzzja37t7XtTti1H7Gt39yBC1FwGpHP15971vKTBcyAbYH3W/ldd/X0P+nYjIyVE4E5HXiISNPrOCbQS9K//k7nOJnOK6qW9smbvf7e4XBs91IqHpKO5eSSQAXM7AU5q4+1Z3fz8wJXj+vWaWfaxi3b2eSC/W4J6iI0BW1ONpx3qdESqNut//eyESpD45KBxluvuz0aUe43UH/M7NzIL32jsKNQPkD/o99tVeR6SXa/Dn3fe+e4ic7j0hx/o7EZETp3AmMrGkmllG1C2FyCnGfzGzYjMrAr4E/ArAzK4ys/IgPDQTOZ3Za5F5uC4NesPaifQ69R7jfe8G/gG4iMjpQ4LX/5CZFQc9R03B5mO9Tp9/JXIKcHHUtnXARWY2y8xygVtG8DrH8xkzyzez0qD+e4LtdwK3mNmSoB25wXitkfotcKWZvcXMUomM2eoAnj32007I7WaWZmZvAq4CfufuPcF7f9XMcoLTsDcRfN7AT4GbzexsiygfdKp2SMP9nYxiW0QmFIUzkYnlASJBqu92G/AVYC2wAXgFeCnYBjAfeAQ4TGTw/L+7+2NExpt9g0hPzH4iPV/HCkP/Q2Tw+aPuXhe1fQWw0cwOE7k44LphxmgN4O6HgG8RGcjft+3/iISnDcCLwOrjvc4I/G/wWuuIjNv6WfBevyfS0/cbMzsEvEqkZ3BE3H0z8CEig/PrgKuBq9298wRq67uas+/2YtS+/UAjkd6yXwM3uPumYN+niPQybgeeJhKcfx7U9TsiF13cDbQAfyDqd3wMw/2diMhJMI3ZFBFJHGZ2CfArdy8JuxYROTnqORMRERGJIwpnIiIiInFEpzVFRERE4oh6zkRERETiiMKZiIiISBxJCbuA0VJUVORz5swJuwwRERGR43rxxRfr3L14qH0JE87mzJnD2rVrwy5DRERE5LjMbNdw+3RaU0RERCSOKJyJiIiIxBGFMxEREZE4onAmIiIiEkcUzkRERETiiMKZiIiISBxROBMRERGJIwpnIiIiInFE4UxEREQkjiicjVBPr/OrP+9if3N72KWIiIhIAlM4G6F9zW2sXF3JN/5YFXYpIiIiksAUzkaoJD+LT7xpLn9Y9xov7moIuxwRERFJUDENZ2a2wsw2m1m1mX1uiP3pZnZPsP95M5sTtW+pmT1nZhvN7BUzy4hlrSPxN5fMY+rkdG5fVUlvr4ddjoiIiCSgmIUzM0sGfgRcDlQA7zezikGHXQ80uns58F3gm8FzU4BfATe4+xLgEqArVrWOVHZ6CrdcvpgNNc3c+1JN2OWIiIhIAoplz9lyoNrdt7t7J/Ab4NpBx1wL/DK4fy/wFjMz4G3ABndfD+Du9e7eE8NaR+zaM2Zw1qw8vvXgZlraQ8+LIiIikmBiGc5mAnuiHtcE24Y8xt27gWagEFgAuJk9ZGYvmdk/x7DOE2Jm3HbNEuoOd/DDR6vDLkdEREQSTLxeEJACXAh8MPj5TjN7y+CDzOwTZrbWzNbW1taOWXFLS/J479kl/PyZHWyvPTxm7ysiIiKJL5bhbC9QGvW4JNg25DHBOLNcoJ5IL9uT7l7n7q3AA8BZg9/A3X/i7svcfVlxcXEMmjC8z6xYSHpKMl+9X1NriIiIyOiJZThbA8w3szIzSwOuA+4bdMx9wEeD++8BHnV3Bx4CTjOzrCC0XQxUxrDWEzYlJ4NPXVrOnzYd5PHNB8MuR0RERBJEzMJZMIbsRiJBqwr4rbtvNLOVZnZNcNjPgEIzqwZuAj4XPLcR+FciAW8d8JK73x+rWk/WX15QRllRNitXV9LV0xt2OSIiIpIALNJRNf4tW7bM165dO+bv+6eqA1z/y7V88aoKrr+wbMzfX0RERMYfM3vR3ZcNtS9eLwgYNy5dNIWLFxTzvUe2UH+4I+xyREREZJxTODtFZsYXr6qgrbOHOx7eEnY5IiIiMs4pnI2C8imT+Oj5c/jNmt28urc57HJERERkHFM4GyV//5b55GelsXJVJYkyjk9ERETGnsLZKMnNTOXmty3khZ0NrN6wL+xyREREZJxSOBtF7zunlIrpk/n6A1W0dcbFUqAiIiIyziicjaLkpMi6m681t3PnE9vCLkdERETGIYWzUba8rICrlk7nzie2sbepLexyREREZJxROIuBW65YjBl87QGtuykiIiInRuEsBmbmZXLDxfO4f8M+nt9eH3Y5IiIiMo4onMXIJy+ax4zcDG5bVUlPr6bWEBERkZFROIuRzLRkPn/lYqr2HeKeNXvCLkdERETGCYWzGLrytOksLyvgjoc309zaFXY5IiIiMg4onMWQmXHr1RU0tnby/T9tDbscERERGQcUzmJsyYxcrjtnFnc9t5Pqgy1hlyMiIiJxTuFsDNz8tgVkpiVzu9bdFBERkeNQOBsDhZPS+fRlC3hqax1/qjoYdjkiIiISxxTOxshHzpvNvOJsvnJ/JR3dWndTREREhqZwNkZSk5P40tVL2Fnfyn89szPsckRERCROKZyNoYsXFHPZ4in84E9bOdjSHnY5IiIiEocUzsbYF66soLOnl289uDnsUkRERCQOKZyNsbKibD5+YRn3vljD+j1NYZcjIiIicUbhLAQ3vrmcoknp3LZqo6bWEBERkQEUzkKQk5HKZ1cs5OXdTfxh3d6wyxEREZE4onAWknefVcLpJbl844+bONLRHXY5IiIiEicUzkKSlGR86eolHDjUwb8/Xh12OSIiIhInFM5CdPbsfN555kz+86kd7K5vDbscERERiQMKZyH77IpFpCQZX32gMuxSREREJA4onIVsWm4Gf/fmch7aeIBnquvCLkdERERCpnAWB66/sIzSgkxWrqqku6c37HJEREQkRApncSAjNZkvXFHB5gMt3P3C7rDLERERkRApnMWJty+ZyvnzCvnOw1toPNIZdjkiIiISEoWzOGFm3Hr1Elrau/juI1vCLkdERERConAWRxZOy+FD587mV3/exab9h8IuR0REREKgcBZnbnrrAiZnpnL7fZVad1NERGQCimk4M7MVZrbZzKrN7HND7E83s3uC/c+b2Zxg+xwzazOzdcHtzljWGU/ystK46a0LeG57PQ9t3B92OSIiIjLGYhbOzCwZ+BFwOVABvN/MKgYddj3Q6O7lwHeBb0bt2+buZwS3G2JVZzz6wPJZLJyaw1fur6K9qyfsckRERGQMxbLnbDlQ7e7b3b0T+A1w7aBjrgV+Gdy/F3iLmVkMaxoXUpKTuPXqCmoa2/jpU9vDLkdERETGUCzD2UxgT9TjmmDbkMe4ezfQDBQG+8rM7GUze8LM3hTDOuPS+eVFrFgyjR89to39ze1hlyMiIiJjJF4vCNgHzHL3M4GbgLvNbPLgg8zsE2a21szW1tbWjnmRsfaFKxfT4843/lgVdikiIiIyRmIZzvYCpVGPS4JtQx5jZilALlDv7h3uXg/g7i8C24AFg9/A3X/i7svcfVlxcXEMmhCu0oIsPvGmufxh3Wu8uKsh7HJERERkDMQynK0B5ptZmZmlAdcB9w065j7go8H99wCPurubWXFwQQFmNheYD0zIwVd/c8k8pk5O5/ZVlfT2amoNERGRRBezcBaMIbsReAioAn7r7hvNbKWZXRMc9jOg0MyqiZy+7Jtu4yJgg5mtI3KhwA3uPiG7jrLTU7jl8sVsqGnm3pdqwi5HREREYswSZaLTZcuW+dq1a8MuIybcnXf/x7PsbmjjsZsvJicjNeySRERE5BSY2YvuvmyoffF6QYBEMTNuu2YJdYc7+OGj1WGXIyIiIjGkcDZOLC3J471nl/DzZ3awo+5I2OWIiIhIjCicjSOfWbGQ9JRkvrK6MuxSREREJEYUzsaRKTkZfOrScv606SCPbz4YdjkiIiISAwpn48xfXlBGWVE2X15dSVdPb9jliIiIyChTOBtn0lKS+JcrF7Ot9gh3Pbcr7HJERERklCmcjUOXLprCxQuK+d4jW6g/3BF2OSIiIjKKFM7GITPji1ctpq2zhzse3hJ2OSIiIjKKFM7GqfIpOXzkvDn8Zs1uXt3bHHY5IiIiMkoUzsaxf7hsPvlZaaxcVUmirPQgIiIy0SmcjWO5manc/LaFvLCzgdUb9oVdjoiIiIwChbNx7n3nlFIxfTJff6CKts6esMsRERGRU6RwNs4lJ0XW3XytuZ07n9gWdjkiIiJyihTOEsDysgKuWjqdO5/Yxt6mtrDLERERkVOgcJYgbrliMWbwtQeqwi5FREREToHCWYKYmZfJDRfP4/4N+3h+e33Y5YiIiMhJUjhLIJ+8aB4zcjO4bVUlPb2aWkNERGQ8UjhLIJlpyXz+ysVU7TvEPWv2hF2OiIiInASFswRz5WnTWV5WwB0Pb6a5tSvsckREROQEKZwlGDPj1qsraGzt5Pt/2hp2OSIiInKCFM4S0JIZuVx3zizuem4n1Qdbwi5HREREToDCWYK6+W0LyExLZuXqKq27KSIiMo4onCWowknpfPqyBTy5pZZHNx0MuxwREREZIYWzBPaR82YzrzibL6+upKNb626KiIiMBwpnCSw1OYkvXb2EnfWt/OKZnWGXIyIiIiOgcJbgLl5QzGWLp/CDR6s52NIedjkiIiJyHApnE8AXrqygo7uHbz+4OexSRERE5DgUziaAsqJsPn5hGb97sYb1e5rCLkdERESOQeFsgrjxzeUUTUrntlUbNbWGiIhIHFM4myByMlL57IqFvLy7iT+s2xt2OSIiIjIMhbMJ5N1nlbC0JJdv/HETRzq6wy5HREREhqBwNoEkJRm3Xr2EA4c6+PfHq8MuR0RERIagcDbBnD07n3eeOZP/fGoHu+tbwy5HREREBlE4m4A+u2IRKUnGVx+oDLsUERERGUThbAKalpvB3725nIc2HuCZ6rqwyxEREZEoCmcT1PUXllFakMnKVZV09/SGXY6IiIgEYhrOzGyFmW02s2oz+9wQ+9PN7J5g//NmNmfQ/llmdtjMbo5lnRNRRmoyX7iigs0HWrj7hd1hlyMiIiKBmIUzM0sGfgRcDlQA7zezikGHXQ80uns58F3gm4P2/yvwx1jVONG9fclUzp9XyHce3kLjkc6wyxERERFi23O2HKh29+3u3gn8Brh20DHXAr8M7t8LvMXMDMDM3gHsADbGsMYJzSwytUZLexfffWRL2OWIiIgIsQ1nM4E9UY9rgm1DHuPu3UAzUGhmk4DPArfHsD4BFk7L4UPnzuZXf97Fpv2Hwi5HRERkwovXCwJuA77r7oePdZCZfcLM1prZ2tra2rGpLAHd9NYFTM5M5fb7KrXupoiISMhiGc72AqVRj0uCbUMeY2YpQC5QD7wR+JaZ7QQ+DXzezG4c/Abu/hN3X+buy4qLi0e/BRNEXlYaN711Ac9tr+ehjQfCLkdERGRCi2U4WwPMN7MyM0sDrgPuG3TMfcBHg/vvAR71iDe5+xx3nwN8D/iau/8whrVOeB9YPouFU3P46gOVtHf1hF2OiIjIhBWzcBaMIbsReAioAn7r7hvNbKWZXRMc9jMiY8yqgZuAo6bbkLGRkpzErVdXsKehjZ89vSPsckRERCYsS5QxRsuWLfO1a9eGXca4d8N/v8iTW2t59J8uYVpuRtjliIiIJCQze9Hdlw21L14vCJCQfOHKxXT3Ot98cFPYpYiIiExICmcyQGlBFp9401x+//JeXtzVGHY5IiIiE47CmRzlby6Zx9TJ6dy+aiO9vYlx2ltERGS8UDiTo2Snp3DL5YvZUNPMvS/VhF2OiIjIhKJwJkO69owZnDUrj289uJmW9q6wyxEREZkwFM5kSGbGbdcsoe5wBz98tDrsckRERCYMhTMZ1tKSPN57dgk/f2YHO+qOhF2OiIjIhKBwJsf0mRULSU9J5iurK8MuRUREZEJQOJNjmpKTwacuLedPmw7y+OaDYZcjIiKS8BTO5Lg+dsEc5hRm8eXVlXT19IZdjoiISEJTOJPjSk9J5otXVbCt9gh3Pbcr7HJEREQSmsKZjMili6Zw0YJivvfIFuoPd4RdjoiISMJSOJMRMTO+dNVi2jp7uOPhLWGXIyIikrAUzmTEyqfk8JHz5vCbNbt5dW9z2OWIiIgkJIUzOSH/cNl88rPSWLmqEnetuykiIjLaRhTOzCzbzJKC+wvM7BozS41taRKPcjNTufltC3lhZwOrN+wLuxwREZGEM9KesyeBDDObCTwMfBj4RayKkvj2vnNKqZg+ma8/UEVbZ0/Y5YiIiCSUkYYzc/dW4F3Av7v7e4ElsStL4llyUmTdzdea2/nxk9vCLkdERCShjDicmdl5wAeB+4NtybEpScaD5WUFXLV0Onc+sY29TW1hlyMiIpIwRhrOPg3cAvze3Tea2VzgsdiVJePBLVcsBuDrD1SFXImIiEjiGFE4c/cn3P0ad/9mcGFAnbv/fYxrkzg3My+TGy6ex+oN+3h+e33Y5YiIiCSEkV6tebeZTTazbOBVoNLMPhPb0mQ8+ORF85iRm8Htqyrp6dXUGiIiIqdqpKc1K9z9EPAO4I9AGZErNmWCy0xL5vNXLqZy3yHuWbMn7HJERETGvZGGs9RgXrN3APe5exegbhIB4MrTprO8rIA7Ht5Mc1tX2OWIiIiMayMNZz8GdgLZwJNmNhs4FKuiZHwxM269uoLG1k6+/8jWsMsREREZ10Z6QcC/uftMd7/CI3YBb45xbTKOLJmRy3XnzOKu53ZSfbAl7HJERETGrZFeEJBrZv9qZmuD23eI9KKJ9Lv5bQvITEtm5eoqrbspIiJykkZ6WvPnQAvwF8HtEPBfsSpKxqfCSel8+rIFPLmllkc3HQy7HBERkXFppOFsnrvf6u7bg9vtwNxYFibj00fOm8284my+vLqSjm6tuykiInKiRhrO2szswr4HZnYBoDV75CipyUl86eol7Kxv5RfP7Ay7HBERkXEnZYTH3QDcZWa5weNG4KOxKUnGu4sXFHPZ4in84NFq3nnWTKbkZIRdkoiIyLgx0qs117v76cBSYKm7nwlcGtPKZFz7wpUVdHT38O0HN4ddioiIyLgy0tOaALj7oWClAICbYlCPJIiyomw+fkEZv3uxhvV7msIuR0REZNw4oXA2iI1aFZKQbry0nKJJ6dy2aqOm1hARERmhUwln+tdWjiknI5V/XrGQl3c38Yd1e8MuR0REZFw4ZjgzsxYzOzTErQWYcbwXN7MVZrbZzKrN7HND7E83s3uC/c+b2Zxg+3IzWxfc1pvZO0+yfRKy95xVwtKSXL7xx00c6egOuxwREZG4d8xw5u457j55iFuOux/zSk8zSwZ+BFwOVADvN7OKQYddDzS6eznwXeCbwfZXgWXufgawAvixmY30ylKJI0lJxq1XL+HAoQ7+/fHqsMsRERGJe6dyWvN4lgPVwaS1ncBvgGsHHXMt8Mvg/r3AW8zM3L3V3fu6WTLQKdRx7ezZ+bzzzJn851M72F3fGnY5IiIicS2W4WwmsCfqcU2wbchjgjDWDBQCmNkbzWwj8ApwQ1RYk3HosysWkZJkfO2BqrBLERERiWuxDGenxN2fd/clwDnALWZ21EymZvaJvsXYa2trx75IGbFpuRn83ZvLeXDjfp6trgu7HBERkbgVy3C2FyiNelwSbBvymGBMWS5QH32Au1cBh4E3DH4Dd/+Juy9z92XFxcWjWLrEwvUXllFakMntqyrp7ukNuxwREZG4FMtwtgaYb2ZlZpYGXAfcN+iY+3h9Gaj3AI+6uwfPSQEws9nAImBnDGuVMZCRmswXrqhg84EW7n5hd9jliIiIxKWYhbNgjNiNwENAFfBbd99oZivN7JrgsJ8BhWZWTWTFgb7pNi4E1pvZOuD3wN+6u86FJYC3L5nK+fMK+c7DW2g80hl2OSIiInHHEmXm9mXLlvnatWvDLkNGYPP+Fi7//pN86NzZrLz2qLPVIiIiCc/MXnT3ZUPti9sLAiRxLZyWw4fOnc2v/ryLTfsPHf8JIiIiE4jCmYTiprcuYHJmKitXVWrdTRERkSgKZxKKvKw0bnrrAp7dVs9DGw+EXY6IiEjcUDiT0Hxg+SwWTs3hqw9U0t7VE3Y5IiIicUHhTEKTkpzErVdXsKehjZ89vSPsckREROKCwpmE6vzyIlYsmcaPHqtmf3N72OWIiIiETuFMQveFKxfT3et888FNYZciIiISOoUzCV1pQRafeNNcfv/yXl7c1Rh2OSIiIqFSOJO48DeXzGPq5HRuX7WR3l5NrSEiIhOXwpnEhez0FG65fDEbapq596WasMsREREJjcKZxI1rz5jBWbPy+NaDm2lp7wq7HBERkVAonEncMDNuvXoJdYc7+OGj1WGXIyIiEgqFM4krp5fm8d6zS/j5MzvYUXck7HJERETGnMKZxJ3PrFhIekoyX1ldGXYpIiIiY07hTOLOlJwMPnVpOX/adJDHNx8MuxwREZExpXAmceljF8xhTmEWX15dSVdPb9jliIiIjBmFM4lL6SnJfPGqCrbVHuGu53aFXY6IiMiYUTiTuHXpoilctKCY7z2yhfrDHWGXIyIiMiYUziRumRlfumoxbZ093PHwlrDLERERGRMKZxLXyqfk8JHz5vCbNbvZ+Fpz2OWIiIjEnMKZxL1/uGw++Vlp3H5fJe5ad1NERBKbwpnEvdzMVG5+20Je2NnA/a/sC7scERGRmFI4k3HhfeeUUjF9Ml+7v4q2zp6wyxEREYkZhTMZF5KTjNuuWcJrze38+MltYZcjIiISMwpnMm4sLyvgqqXTufOJbextagu7HBERkZhQOJNx5ZYrFgPw9QeqQq5EREQkNhTOZFyZmZfJDRfPY/WGfTy/vT7sckREREadwpmMO5+8aB4zcjO4fVUlPb2aWkNERBKLwpmMO5lpyXz+ysVU7jvEPWv2hF2OiIjIqFI4k3HpytOms7ysgDse3kxzW1fY5YiIiIwahTMZl8yMW6+uoLG1k+8/sjXsckREREaNwpmMW0tm5HLdObO467mdVB9sCbscERGRUaFwJuPazW9bQGZaMitXV2ndTRERSQgKZzKuFU5K59OXLeDJLbU8uulg2OWIiIicMoUzGfc+ct5s5hVn8+XVlXR0a91NEREZ3xTOZNxLTU7iS1cvYWd9K794ZmfY5YiIiJySmIYzM1thZpvNrNrMPjfE/nQzuyfY/7yZzQm2v9XMXjSzV4Kfl8ayThn/Ll5QzFsWTeEHj1ZzsKU97HJEREROWszCmZklAz8CLgcqgPebWcWgw64HGt29HPgu8M1gex1wtbufBnwU+O9Y1SmJ41+uqqCju4dvP7g57FJEREROWix7zpYD1e6+3d07gd8A1w465lrgl8H9e4G3mJm5+8vu/lqwfSOQaWbpMaxVEkBZUTYfv6CM371Yw/o9TWGXIyIiclJiGc5mAtFr69QE24Y8xt27gWagcNAx7wZecveOGNUpCeTGS8spmpTO7as2amoNEREZl+L6ggAzW0LkVOcnh9n/CTNba2Zra2trx7Y4iVuMdZYAAB/ISURBVEs5Gan884qFvLS7if9d99rxnyAiIhJnYhnO9gKlUY9Lgm1DHmNmKUAuUB88LgF+D3zE3bcN9Qbu/hN3X+buy4qLi0e5fBmv3nNWCUtLcvn6H6s40tEddjkiIiInJJbhbA0w38zKzCwNuA64b9Ax9xEZ8A/wHuBRd3czywPuBz7n7s/EsEZJQElJxq1XL+HAoQ7+4/Ehc72IiEjcilk4C8aQ3Qg8BFQBv3X3jWa20syuCQ77GVBoZtXATUDfdBs3AuXAl8xsXXCbEqtaJfGcPTufd545k588tZ3d9a1hlyMiIjJiliiDppctW+Zr164NuwyJI/ub27n0O49z0fxi7vzw2WGXIyIi0s/MXnT3ZUPti+sLAkROxbTcDP7uzeU8uHE/z1bXhV2OiIjIiCicSUK7/sIySgsyuX1VJd09vWGXIyIiclwKZ5LQMlKT+cIVFWw+0MLdL+wOuxwZZ9ydju6esMsQkQkmJewCRGLt7Uumcv68Qr7z8BauXjqD/Oy0sEuSOFV3uIMNNU2s39PMhpomNtQ009jaydKSPN40v4gLyos4c1Ye6SnJYZcqIglMFwTIhLB5fwuXf/9JPnTubFZe+4awy5E40NLexSt7m9lQ09wfyPY2tQFgBvOnTGJpSR7FOem8sKOBdXua6Ol1MlOTWV5WwIXlkbC2aFoOSUkWcmtEZLw51gUB6jmTCWHhtBw+dO5sfvXnXXzgjbNYNG1y2CXJGGrv6qFy3yE27In0hq2vaWJ73RH6/t90VkEWZ87K42Pnz2FpSS5vmJlLdvrAr8eW9i6e397A09V1PF1dx1cfqAKgMDuN88uLuLC8kAvKiyjJzxrr5olIglHPmUwYTa2dXHLH41RMn8yv/+qNmKm3IxF19/Sy9eDhSG9Y0Cu2aV8L3b2R77rinHROL8llaUkeS4OfBSdxqnt/czvPVNfxTBDWDrZElv+dU5jFhfOLuLC8iPPmFpGblTqq7RORxHCsnjOFM5lQ7npuJ1/6343c+aGzWfGGaWGXI6fI3dlV38r6YHzYhpomXt17iLauyCD+nIyU/gB2ekkep5fmMm1yxqgHc3en+uBhng7C2nPb6jnS2UOSwWkzc7mgPBLWzpqdT0aqxquJiMKZSL/unl6u/Lenae3q5v/+8WL9QznOHDjUzro9Tf2D9TfUNNPc1gVAekoSb5iZy9KSXE4PesXmFGaHMh6sq6eX9Xua+sPay7ub6O510lOSBoxXq5g+WePVRCYohTORKM9W1/GBnz7PZ96+kL97c3nY5cgwmlo7Xx+sH/w8cChy6jA5yVg4NYfTS18/Pblgag6pyfE5O9Dhjm5e2FHP01vreaa6js0HWgDIz0oNxqtFbqUFGq8mMlHoggCRKOeXF7FiyTR+9Fg17z6rhGm5GWGXNOG1dnaz8bVDrI8asL8rak3UuUXZnDe3MHJ6sjSPJTMmj6tez0npKVy6aCqXLpoKwMFD7Tyzra4/rN2/YR8QuTCh7xTo+fMKNe2LyASlnjOZkHbXt3LZd5/gytOm8933nRF2ORNKZ3cvm/e3BOPEImFsy4EWgvH6zMjNiPSGlUZOT75hZi65mYk7qN7d2VZ7pP/Cgj9vq6eloxszeMOM18erLZuj8WoiiUSnNUWG8O2HNvGjx7bx//7mfM6enR92OQmpt9fZXne4f1LX9TXNVO47RGd3ZCmt/KzUYLB+bn8gm5IzsXsyu3t6WV/T3B/WXt7dSFePk5aSxDlz8vvD2pIZuSRrvJrIuKVwJjKEIx3dXPqdx5k6OYM//O0FGph9itydvU1t/aclN+xp5pW9zRzu6AYgKy2ZN8zM7Q9iZ5TmUZKfqSlNjuNIRzcv7Gzgma2RsLZpf2S8Wl5WKufPK+wPa7MKsvS7FBlHFM5EhvH7l2v4x3vW8+33LOW9y0rDLmdcqT/cwYaa5gFXT9Yf6QQgNdlYPH1y/1WTp5fmMa94knp6RkFtSwfPbqvj6a2RK0Ffa24HoCQ/s/8q0PPnFVI4KT3kSkXkWBTORIbh7rz7P55ld0Mbj918MTkZiTu26VSMdKmjvl6xRdNztP7kGHB3dtS9Pl7t2W31tLRHeiorpk/mwmA90OVzCshM0+chEk8UzkSOYf2eJq790TN88uK53HL54rDLCV17Vw9V+w71n55cv2fgUkelBZn9QaxvwP7gpY4kHN09vbz62iGe3lrL09V1vLSric6eXtKSkzh7dn5/WDttpsariYRN4UzkOD7zu/X8Yd1eHv7Hiykryg67nDHT3dNLde1h1u95fS6xzftb6OqJfC8UTUrnjNJTX+pIwtHa2c2anY2RnrWtdVTuOwTA5IwUzptX2H8atKwoW+PVRMaYwpnIcRxsaefSO57g3LkF/PSj54RdTkyc2FJHkZ/Tc0d/qSMJT/3hDp7dFplb7amtdf2npmfmZXJBsHD7+fOKKM7ReDWRWFM4ExmBHz+xja//cRO//PhyLl5QHHY5p+zAofYBk7q+sreZptbXlzpaMmNyMKlrJIiVhbTUkYTD3dnd0MpTwYUFz26r718Ka9G0nEiv2vwi3lhWQFaaTluLjDaFM5ER6Oju4e3ffZLkJOPBT18Ut0sBDaW5tYsNe5sGnJ6MXupowdScAacn43mpIwlHT6+z8bXm/vVA1+xspLO7l9Rk48xZ+ZElpuYXsXRmLin62xE5ZQpnIiP0p6oDXP/LtXzxqgquv7As7HKG1NbZw6uvNff3im2oaWLnoKWO+k9PluZSMT1XV+rJCWvv6mHtzkaeqq7lmeo6Nr52CHfISU/h3KjxavOKNV5N5GQonImMkLvz0f9aw8u7G3n85ktCnyuqqydqqaM9kdOT0UsdTc/NiBonlsdpJYm91JGEp+FIJ89tq+/vWdvdEPkfgmmTMyIT4c6PjFmb6Cs8iIyUwpnICag+2MKK7z3Fe5eV8vV3nTZm7xtZ6uhI0CN29FJHecFSR2doqSOJA7vrW4PF2+t4Zltd/3jGhVNz+sPa8rJCJmmaFZEhKZyJnKCVqyr5r2d3sPpTF7JkRu6ov/5QSx29ureZlmGWOjq9JI/SAi11JPGpt9ep3Heov1fthR0NdHT3kpJknDkrr3+JqdNL8zTWUSSgcCZygprbunjzHY9TXjyJez557imHor6ljqKnsag7PHCpo+jTk+VTtNSRjF/tXT28tKuRp4OVC17Z24w7ZKclc+7cQi6cHwlr5VMm6X84ZMJSOBM5CXc/v5vP//4VfviBM7lq6YwRP+9wRzev9Aexo5c6Ki+eFCz8raWOZGJoah04Xq3vApYpOen9FxZcUF7EtFydppeJQ+FM5CT09DpX/+Bpmlo7+dM/XTLkFY+DlzraUNPMttrDQy51tDRY6khjcOJPr/fS3N5MfVs9DW0N1LcGP9vq6erpYn7hfBYXLWZu/lySkxSkT9WehtbI4u3VkQlxG45EepHLp0yKTNlRXsQb5xZorVtJaApnIifphR0N/MWPn+PTl83nU5fOZ+vBlv6rJjfUNLNp/6EBSx2dHjVY/3QtdTTm3J1DHYf6g9XgoNXQ1jDkvsb2Rnq997ivn56czoLCBVQUV7C4aDGLixdTUVzB/IL5pKdoVv2T0dvrVO0/FCzeXs8LO+pp7+olOck4o/T18WpnlOaRlqLxapI4FM5ETsGNd7/EQxv3k5KU9PpSR+kpnBYEsb7Tk1rqaPS4O0e6jhwVrPoeD9g2aF+P9wz7ujlpORRmFVKQWUBBZgGFmYUDf2Yd/TjZktlcv5mq2ioqayupqquiqq6KHY07cCLfn8mWzLyCeSwuWjwguC0qWsSktElj9WtLCB3dPby0qykIa3VsqGmi1yMXybyxrCC4ErSIhVNz9N+bjGsKZyKnYF9zG1/8w0ZK8jO11NFJaO1qHb4Ha3D4itrX2dM57Gtmp2YPCFMjCVoFmQWkJo/eabLWrla21G+JBLbaKirrIj+3Nmylu7e7/7hZubNeD2x94a14MQWZBaNWSyJrbu3iue2R05/PVNexve4IEOmpvjBYD/SC8iJm5GWGXKnIiVE4E5FT1tHdcexTha31NLQfva+9u33Y18xIyRgQngqzCinIGLoHq+9xfmY+GSnxO3C8q6eL6oZqquqietpqq9hUt4m27rb+46ZmT42cFi2KhLW+4DZt0jT1CB3D3qa2/qD2THVd/1XPc4uz+y8uOHduoSZjlrincCYi/Tp7OmlsazyhU4X1bfW0drUO+5qpSakUZhUODFrHOFXY9zgzdeL0dvR6L7uadg0IbH29bc0dzf3H5abnHjWmbXHRYmbnzSbJNOYqmruz+UALT2+NnAJ9fnsDbV09JBksLcnjTfMjYe3MWXm6IlrijsLZKOjs6eQ/1vwHaclp/bf0lPSBj5PTR7Q/JSlF/2csp6y7t5vGtsYTHpPV0tky7GsmW/IJj8kqyCwgO1XrK54sd2ff4X1U1VYN6G2rrK3k4JGD/cdlpmSyqGjRUcFtXv68UT1dO551dvfy8u7G/vFq62ua6el1MlOTWV5W0N+ztmhajoYlSOgUzkZBY1sjBd8avTEiJxLmjtqfdOoBUQEyfvR6L03tTSc0Jqu+tX5Ab8tgSZZEfkb+0IHqGEErJ02DrONJQ1vDgAsR+n7ubt7df0xqUmr/VB/RY9oWFi6cUD2TQznU3sXz2xt4prqOp7bWsq02Ml6tMDuN88uL+sesleRnhVypTEShhTMzWwF8H0gGfuru3xi0Px24CzgbqAfe5+47zawQuBc4B/iFu994vPeKdTjrmweps6eTjp4OOns6+28d3YMen8z+3s6Tfs2Ono6YtHmoMDfiwDeKAXK4Y+ItQLo7zR3NJzwmq7Gtsf+qv6H0hawTCVq5Gbk6BZbADnceZlPdpqOC27bGbf1TghhGWX7ZkBcjTE6fHHILwrGvuY1ngrnVnq6uo7Yl8t05pzCrf9WC8+YWkZulnkiJvVDCmZklA1uAtwI1wBrg/e5eGXXM3wJL3f0GM7sOeKe7v8/MsoEzgTcAb4iHcBbP3J3u3u4TCognFSJP8PjoY2IZIE+6x/AYAXLwc5KTkgdOUjrEmKzGtsZjTuMwOX3yCQ9+z8vI06SnMmId3R1sbdh61BWkm+s3D7j6dWbOzAEXIfT9LM4uDrH6seXubD14OLJwe3Udf95ez5HOyHi102bmckF5Ecvm5FOan8XM/Eyy0jR5tIyusMLZecBt7v724PEtAO7+9ahjHgqOec7MUoD9QLEHRZnZx4BlCmfjn7vT4z2j08t4vJB5kq850gA5KW3SCY/Jys/I17ggCU13bzc7GnccNaatqraKI11H+o8rzCwc8mKEksklcdVLHQtdPb2s29PUH9Ze3tNET+/r/z4WZKcxMy+TkvzM13/mZwU/M5ms1QzkBB0rnMXyfwVmAnuiHtcAbxzuGHfvNrNmoBCoi2FdEgIzI8VSSElLIZvssMsZ0nABsrOnk67eLvIy8sjPyNdM8DLupCSlML9wPvML53PNwmv6t7s7NYdqjhrTdm/VvTS81NB/XE5aDouKFg2Y+qOiuIKyvLKE6dlNTU7inDkFnDOngH986wJa2rvYcqCFmsa2/tvepjY2H2jh0U0H6egeuKLE5IwUSoJettcDXCS8leRnkpuZmvABV0bPuO6nNbNPAJ8AmDVrVsjVyHg3HgKkyGgyM0pzSynNLeXt5W/v3+7u1LbWHrUqwiPbH+Gu9Xf1H5eenM7CooVHjWlLhOWscjJSOXt2AWfPPnqfu1N3uJO9TW3UNLaytz/AtbKr/gjPVNfR2jlwiMOk9JSoHre+APd6z1thdprCm/SLZTjbC5RGPS4Jtg11TE1wWjOXyIUBI+LuPwF+ApHTmqdUrYiIAJHQNiV7ClOyp3DxnIsH7Gtub+6fp62vt23Na2v47cbfHnM5q4riChYVLSI7bfz/j4+ZUZyTTnFOOmeU5h21391pau0Kettaj+p9e2FnAy3t3QOek5GaFOl5GxDgIo9L8zMpmpSu6T8mkFiGszXAfDMrIxLCrgM+MOiY+4CPAs8B7wEe9VhePioiIqckNyOXc0vO5dyScwdsH7ycVV9wu3/r/QOWs5qdO/uoixESbTkrMyM/O4387DROK8kd8pjmtq6gx6016IFrizxuamVDTRONrV0Djk9LTmJm1Hi3wQFu6uQMkhXeEkasp9K4Avgekak0fu7uXzWzlcBad7/PzDKA/yZyZWYDcJ27bw+euxOYDKQBTcDboq/0HEwXBIiIxJ/o5ayiryA93nJWfcFtoi5ndaSje4jTpm3UNLWxt7G1f9mqPilJxvS8DEryhh73Ni03g9RkTa8TTzQJrYiIxJW+5ayirxwdajmrvIy8o8a0VRRXMCt31oSey6+ts4e9TW1HBbi+xwcODbz6PMlgem7msOPepudlaImrMaZwJiIi44K7s//w/qOuIK2qreLAkQP9x2WlZrGwcOGAMW2LixdrOatAR3cP+5raB4x7iw5w+5rbiJopBDOYkpM+7Li3kvxMMlIV3kaTwpmIiIx7J7qcVXRwW1C4YMIvZxWtq6eX/c3t/VeZDh73tq+pne7egfmgaFJ6f49byRABLjt9XE8AMeYUzkREJGENtZxVVV0V1Q3Vwy5nFX0xwkRdzupYenqdA4fahxz3trcpEuI6ewbO9ZaflRoJa1FThPT3vBVoot7BFM5ERGTCGbycVV9v23DLWQ2+GGEiLWd1onp7nbrDHexpHH7cW3vXwPCW0zdRb9QVpyVRAS4va2JN1KtwJiIiEohezir6QoSquioOdx7uPy4nLYeirCKKsooozCqM/Mwc9DNqe2FWIRkpGSG2LH64O/VHOqMC28BxbzWNrRwZNFFvdlryUePc+h6XJOBEvQpnIiIix9G3nFVfD9vOpp3Ut9VT31pPXWsdda111LfVc6jj0LCvkZ2aPeIw1/c4KzVrDFsZH9yd5rauqAl6B417a2zl0BAT9c7Mi1rTNG9g71vxOJuoV+FMRERklHT2dNLQ1hAJa0Fwq2+rf/1x29Hbm9qbhn29zJTMEwpzRVlFZKdmJ1Qv0lAOtUcm6h08WW/fqdOGIwPnektLTmJGXsbQV5zmZzItzibqVTgTEREJUXdvNw1tDUOHuejHUdsb2hr6l8QaLC057ejQljl8mCvMLGRy+uSECnRHOrp5ren1yXkHj3urbRk411tKkjEtN2PAOLe+AFeanzXmE/UeK5zpulcREZEYS0lK6V+vdKR6entoam8aUZjbeHAjda11NLQ10OM9Q75eSlLK8D1yw2zPzciN28l+s9NTmD81h/lTc4bc397V039l6eBxb09vreNASzvR/VNJBtMmZzAzP5MLyov49GULxqglR1M4ExERiUPJSckUZkUuNBipXu+lub15+DAXddp1c93m/u3R659GS7Kk/osdRnraNT8jn+Sk8CeszUhNZl7xJOYVTxpyf2d3L/uaB16kUBP0xDUOOmU61nRaU0REZAJzd1o6W4YPc0P01NW11g2YjiSaYeRn5p/QadeCzAJSkiZWf5FOa4qIiMiQzIzJ6ZOZnD6ZuflzR/Qcd+dI15ERhbk9h/awbv866lrrBix2P1heRt4JnXYtzCokLTlttH4NcUXhTERERE6ImTEpbRKT0iYxO2/2iJ/X2tU6okC3//D+/nF0R7qODPt6fXPRnchp1/EwF53CmYiIiIyJrNQssnKzKM0tHfFz2rvbqW+tH9EVrn3j6I43F93xwtzc/Lksn7l8NJp8UhTOREREJG5lpGQwc/JMZk6eOeLnnMhcdDuadhw1F90V86/g/g/cH4vmjIjCmYiIiCSUtOQ0pk2axrRJ00b8nOi56MKePkThTERERCa8k5mLLlbic2Y5ERERkQlK4UxEREQkjiiciYiIiMQRhTMRERGROKJwJiIiIhJHFM5ERERE4ojCmYiIiEgcUTgTERERiSMKZyIiIiJxROFMREREJI6Yu4ddw6gws1pg1xi8VRFQNwbvE48mctthYrdfbZ+4JnL7J3LbYWK3fyzaPtvdi4fakTDhbKyY2Vp3XxZ2HWGYyG2Hid1+tX1ith0mdvsnctthYrc/7LbrtKaIiIhIHFE4ExEREYkjCmcn7idhFxCiidx2mNjtV9snronc/oncdpjY7Q+17RpzJiIiIhJH1HMmIiIiEkcUzoZgZivMbLOZVZvZ54bYn25m9wT7nzezOWNfZeyMoP0fM7NaM1sX3P4qjDpjwcx+bmYHzezVYfabmf1b8LvZYGZnjXWNsTKCtl9iZs1Rn/uXxrrGWDGzUjN7zMwqzWyjmf3DEMck5Gc/wrYn8mefYWYvmNn6oP23D3FMQn7nj7DtCft938fMks3sZTNbPcS+cD57d9ct6gYkA9uAuUAasB6oGHTM3wJ3BvevA+4Ju+4xbv/HgB+GXWuM2n8RcBbw6jD7rwD+CBhwLvB82DWPYdsvAVaHXWeM2j4dOCu4nwNsGeLvPiE/+xG2PZE/ewMmBfdTgeeBcwcdk5Df+SNse8J+30e18Sbg7qH+xsP67NVzdrTlQLW7b3f3TuA3wLWDjrkW+GVw/17gLWZmY1hjLI2k/QnL3Z8EGo5xyLXAXR7xZyDPzKaPTXWxNYK2Jyx33+fuLwX3W4AqYOagwxLysx9h2xNW8HkeDh6mBrfBg7ET8jt/hG1PaGZWAlwJ/HSYQ0L57BXOjjYT2BP1uIajv6j6j3H3bqAZKByT6mJvJO0HeHdwaudeMysdm9Liwkh/P4nqvOAUyB/NbEnYxcRCcNriTCK9CNES/rM/RtshgT/74LTWOuAg8H/uPuxnn2jf+SNoOyT29/33gH8GeofZH8pnr3AmJ2MVMMfdlwL/x+v/VyGJ7SUiy42cDvwA+EPI9Yw6M5sE/D/g0+5+KOx6xtJx2p7Qn72797j7GUAJsNzM3hB2TWNlBG1P2O97M7sKOOjuL4Zdy2AKZ0fbC0T/n0FJsG3IY8wsBcgF6sekutg7bvvdvd7dO4KHPwXOHqPa4sFI/j4Skrsf6jsF4u4PAKlmVhRyWaPGzFKJhJNfu/v/N8QhCfvZH6/tif7Z93H3JuAxYMWgXYn8nQ8M3/YE/76/ALjGzHYSGcJzqZn9atAxoXz2CmdHWwPMN7MyM0sjMgDwvkHH3Ad8NLj/HuBRD0YLJoDjtn/QOJtriIxRmSjuAz4SXLl3LtDs7vvCLmosmNm0vrEWZracyPdHQvwDFbTrZ0CVu//rMIcl5Gc/krYn+GdfbGZ5wf1M4K3ApkGHJeR3/kjansjf9+5+i7uXuPscIv/WPeruHxp0WCiffUqs32C8cfduM7sReIjIlYs/d/eNZrYSWOvu9xH5IvtvM6smMoD6uvAqHl0jbP/fm9k1QDeR9n8stIJHmZn9D5Er04rMrAa4lcggWdz9TuABIlftVQOtwF+GU+noG0Hb3wP8jZl1A23AdYnwD1TgAuDDwCvB+BuAzwOzIOE/+5G0PZE/++nAL80smUjo/K27r54g3/kjaXvCft8PJx4+e60QICIiIhJHdFpTREREJI4onImIiIjEEYUzERERkTiicCYiIiISRxTOREREROKIwpmIjCkzczP7TtTjm83stlF67V+Y2XtG47WO8z7vNbMqM3ts0PY5ZtZmZuuibh8Zxfe9xMxWj9briUh80jxnIjLWOoB3mdnX3b0u7GL6mFlKsHbeSFwP/LW7Pz3Evm3BcjgiIidFPWciMta6gZ8A/zh4x+CeLzM7HPy8xMyeMLP/NbPtZvYNM/ugmb1gZq+Y2byol7nMzNaa2ZZg7by+xZ2/bWZrggWcPxn1uk+Z2X1A5RD1vD94/VfN7JvBti8BFwI/M7Nvj7TRZnbYzL5rZhvN7E9mVhxsP8PM/hzU9Xszyw+2l5vZIxZZbPylqDZOssgC1JvM7NdRM/d/w8wqg9e5Y6R1iUj8UTgTkTD8CPigmeWewHNOB24AFhOZ0X6Buy8nst7fp6KOmwMsB64E7jSzDCI9Xc3ufg5wDvDXZlYWHH8W8A/uviD6zcxsBvBN4FLgDOAcM3uHu68E1gIfdPfPDFHnvEGnNd8UbM8mMuv4EuAJIiswANwFfDZYWPqVqO2/Bn4ULDZ+PtC3VNSZwKeBCmAucIGZFQLvBJYEr/OV4/0yRSR+KZyJyJhz90NEQsnfn8DT1rj7vmAR5m3Aw8H2V4gEsj6/dfded98KbAcWAW8jsi7mOuB5oBCYHxz/grvvGOL9zgEed/fa4HTnr4GLRlDnNnc/I+r2VLC9F7gnuP8r4MIgnOa5+xPB9l8CF5lZDjDT3X8P4O7t7t4aVW+Nu/cC64K2NwPtRHrz3kVkeSkRGacUzkQkLN8j0qOVHbWtm+B7ycySgLSofR1R93ujHvcycPzs4DXpHDDgU1GBqczd+8LdkVNqxck72bXzon8PPUDfWLnlwL3AVcCDp1ibiIRI4UxEQuHuDcBviQS0PjuBs4P71xAsvH6C3mtmScEYrbnAZuAhIgt3pwKY2QIzyz7WiwAvABebWVGwMPT7iZyOPFlJRBYQB/gA8LS7NwONUac+Pww84e4tQI2ZvSOoN93MsoZ7YTObBOS6+wNExvKdfgp1ikjIdLWmiITpO8CNUY//E/hfM1tPpPfnZHq1dhMJVpOBG9y93cx+SuT030vBAPpa4B3HehF332dmnwMeI9Lzdr+7/+8I3n9ecPq0z8/d/d+ItGW5mf0LcBB4X7D/o0TGxmUROQ37l8H2DwM/NrOVQBfw3mO8Zw6R31tGUOtNI6hTROKUuZ9sz7qIiIyUmR1290lh1yEi8U+nNUVERETiiHrOREREROKIes5ERERE4ojCmYiIiEgcUTgTERERiSMKZyIiIiJxROFMREREJI4onImIiIjEkf8f+9Nup6gKjH8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "12PYfqX3QrE-",
        "colab_type": "text"
      },
      "source": [
        "## Get Predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ey9FMYJxQqGs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_predictions(model, df, num_labels, device=\"cpu\", batch_size=32):\n",
        "  num_iter = math.ceil(df.shape[0]/batch_size)\n",
        "  \n",
        "  pred_probs = np.array([]).reshape(0, num_labels)\n",
        "  \n",
        "  model.to(device)\n",
        "  model.eval()\n",
        "  \n",
        "  for i in range(num_iter):\n",
        "    df_subset = df.iloc[i*batch_size:(i+1)*batch_size,:]\n",
        "    X = df_subset[\"features\"].values.tolist()\n",
        "    masks = df_subset[\"masks\"].values.tolist()\n",
        "    X = torch.tensor(X)\n",
        "    masks = torch.tensor(masks, dtype=torch.long)\n",
        "    X = X.to(device)\n",
        "    masks = masks.to(device)\n",
        "    with torch.no_grad():\n",
        "      logits = model(input_ids=X, attention_mask=masks)\n",
        "      logits = logits.sigmoid().detach().cpu().numpy()\n",
        "      pred_probs = np.vstack([pred_probs, logits])\n",
        "  \n",
        "  return pred_probs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Be95DEo0Sio_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wo0YyAH6XGTh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "23cd88a4-8201-4358-bdd5-0898260fdb82"
      },
      "source": [
        "num_labels = len(label_cols)\n",
        "pred_probs = generate_predictions(model, test, num_labels, device=\"cuda\", batch_size=32)\n",
        "pred_probs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[8.47498775e-01, 5.28851291e-03, 2.75254071e-01, 1.10191712e-02,\n",
              "        5.73680699e-01, 3.94195458e-03],\n",
              "       [8.08631927e-02, 1.50148233e-03, 1.64668611e-03, 2.93011690e-04,\n",
              "        2.28061038e-03, 1.32608460e-03],\n",
              "       [3.03586453e-01, 1.59935781e-03, 6.83232676e-03, 6.06743561e-04,\n",
              "        1.48619805e-02, 1.07122608e-03],\n",
              "       [2.11075962e-01, 1.79364777e-03, 4.22228873e-03, 5.09508944e-04,\n",
              "        9.63884499e-03, 1.51514553e-03],\n",
              "       [7.49880731e-01, 4.39048093e-03, 9.04312953e-02, 1.97972334e-03,\n",
              "        1.86765105e-01, 2.29417719e-03],\n",
              "       [7.20549345e-01, 2.97644874e-03, 2.73051541e-02, 1.41795538e-03,\n",
              "        5.05878776e-02, 1.58901396e-03],\n",
              "       [4.88679484e-02, 1.12264173e-03, 7.71198364e-04, 1.68590690e-04,\n",
              "        1.61242718e-03, 1.15361123e-03],\n",
              "       [1.69447124e-01, 1.57633889e-03, 3.69325466e-03, 4.03993647e-04,\n",
              "        9.95277520e-03, 1.08596869e-03],\n",
              "       [2.47763231e-01, 1.58293045e-03, 4.74704243e-03, 4.82419360e-04,\n",
              "        8.10387731e-03, 1.19192246e-03],\n",
              "       [4.75599021e-02, 9.53774841e-04, 7.46596896e-04, 1.01950631e-04,\n",
              "        1.06954610e-03, 8.32076301e-04],\n",
              "       [2.19844371e-01, 1.56446325e-03, 3.79159255e-03, 2.75032013e-04,\n",
              "        5.30635053e-03, 1.29934214e-03],\n",
              "       [2.69197196e-01, 1.50415639e-03, 6.60427148e-03, 5.67742507e-04,\n",
              "        1.15309590e-02, 1.26608019e-03],\n",
              "       [8.38159740e-01, 3.70695163e-03, 1.36333719e-01, 2.74790707e-03,\n",
              "        2.56618142e-01, 2.02462496e-03],\n",
              "       [2.20082536e-01, 1.47227850e-03, 3.45048727e-03, 3.14162287e-04,\n",
              "        5.15279127e-03, 1.71114109e-03],\n",
              "       [1.18332952e-01, 1.38426386e-03, 3.22261150e-03, 3.67909641e-04,\n",
              "        4.77491878e-03, 1.46525796e-03],\n",
              "       [7.08023787e-01, 1.83396484e-03, 4.82648350e-02, 1.34891190e-03,\n",
              "        9.86424834e-02, 1.12139701e-03],\n",
              "       [8.90225589e-01, 4.78377799e-03, 3.43031913e-01, 5.57907904e-03,\n",
              "        6.30327642e-01, 3.50026530e-03],\n",
              "       [9.52481031e-02, 1.20664202e-03, 1.53240550e-03, 2.24274132e-04,\n",
              "        1.77109370e-03, 1.12190517e-03],\n",
              "       [2.45676301e-02, 1.23517134e-03, 1.08205073e-03, 1.63562159e-04,\n",
              "        1.96240097e-03, 1.05016865e-03],\n",
              "       [2.10223526e-01, 1.79357000e-03, 3.71645601e-03, 3.42010113e-04,\n",
              "        5.17352438e-03, 9.16733756e-04],\n",
              "       [2.56964445e-01, 1.65279454e-03, 6.58051344e-03, 3.61721410e-04,\n",
              "        9.76551604e-03, 1.40472129e-03],\n",
              "       [3.17823291e-01, 1.56559807e-03, 8.74991622e-03, 4.45094687e-04,\n",
              "        1.60501972e-02, 1.17139344e-03],\n",
              "       [1.20268889e-01, 1.12141413e-03, 9.86477709e-04, 1.67988619e-04,\n",
              "        1.51541107e-03, 1.19839772e-03],\n",
              "       [5.30763030e-01, 2.33473605e-03, 1.58997867e-02, 9.29373957e-04,\n",
              "        2.27748249e-02, 1.75432209e-03],\n",
              "       [8.66215825e-02, 1.41136278e-03, 8.02656694e-04, 1.34393238e-04,\n",
              "        1.16474810e-03, 9.09006165e-04],\n",
              "       [1.83443725e-01, 1.27550529e-03, 3.29439761e-03, 3.21432570e-04,\n",
              "        4.17552143e-03, 8.30576406e-04],\n",
              "       [2.13690609e-01, 1.52399379e-03, 2.37801275e-03, 3.96934542e-04,\n",
              "        4.32812283e-03, 1.43060950e-03],\n",
              "       [9.53698456e-02, 1.74219068e-03, 1.42718526e-03, 2.10572616e-04,\n",
              "        2.38635670e-03, 1.12014846e-03],\n",
              "       [1.41685352e-01, 1.97805930e-03, 1.94099080e-03, 3.15331650e-04,\n",
              "        3.99011560e-03, 1.51108741e-03],\n",
              "       [1.94198027e-01, 1.85292808e-03, 8.35785735e-03, 6.80788129e-04,\n",
              "        1.65828727e-02, 1.40346366e-03],\n",
              "       [4.47143555e-01, 3.08925426e-03, 2.33551916e-02, 1.27889938e-03,\n",
              "        5.45182899e-02, 2.13571894e-03],\n",
              "       [3.80788445e-01, 2.91068596e-03, 3.30772847e-02, 7.82444316e-04,\n",
              "        5.01079261e-02, 2.13985820e-03],\n",
              "       [3.60157222e-01, 2.50548683e-03, 8.99553578e-03, 5.94001904e-04,\n",
              "        1.04422234e-02, 1.65541982e-03],\n",
              "       [4.14277971e-01, 2.47471407e-03, 2.51765233e-02, 7.65795354e-04,\n",
              "        4.29735258e-02, 1.22221757e-03],\n",
              "       [1.11247808e-01, 1.19182980e-03, 1.48371048e-03, 1.72586209e-04,\n",
              "        1.70539611e-03, 9.13101714e-04],\n",
              "       [5.06092347e-02, 1.00152439e-03, 8.30120232e-04, 1.25930994e-04,\n",
              "        1.38447538e-03, 1.08217390e-03],\n",
              "       [7.72732317e-01, 4.69809351e-03, 1.82157516e-01, 5.37445443e-03,\n",
              "        3.61099988e-01, 3.35315499e-03],\n",
              "       [1.41623104e-02, 1.64165325e-03, 4.91973187e-04, 7.02583566e-05,\n",
              "        9.65599902e-04, 1.23348308e-03],\n",
              "       [6.09642603e-02, 1.10703846e-03, 9.90051776e-04, 1.74282744e-04,\n",
              "        1.56768877e-03, 9.60106379e-04],\n",
              "       [5.85209548e-01, 2.16589193e-03, 2.42706537e-02, 1.09416398e-03,\n",
              "        3.30516435e-02, 1.81765854e-03],\n",
              "       [6.03976473e-02, 1.58912968e-03, 6.66991808e-04, 2.14984466e-04,\n",
              "        1.34444854e-03, 1.42348511e-03],\n",
              "       [1.25252411e-01, 1.57427415e-03, 1.96415209e-03, 2.24616524e-04,\n",
              "        3.52493045e-03, 1.29795307e-03],\n",
              "       [9.74716023e-02, 1.06865691e-03, 1.10591063e-03, 1.89609287e-04,\n",
              "        1.55942224e-03, 8.32949649e-04],\n",
              "       [6.37396812e-01, 2.13839440e-03, 2.84559708e-02, 1.15971453e-03,\n",
              "        5.38683906e-02, 1.29919045e-03],\n",
              "       [5.74822426e-01, 2.35030497e-03, 1.60451811e-02, 1.16466882e-03,\n",
              "        3.40496153e-02, 1.09033694e-03],\n",
              "       [9.71721858e-02, 1.92862831e-03, 1.69494364e-03, 3.12046497e-04,\n",
              "        5.38044004e-03, 1.48652331e-03],\n",
              "       [2.65455581e-02, 1.95997162e-03, 6.74192444e-04, 1.13916678e-04,\n",
              "        9.84401326e-04, 1.22159673e-03],\n",
              "       [6.29066110e-01, 2.07978650e-03, 2.43567880e-02, 1.11879664e-03,\n",
              "        3.87666598e-02, 1.49549148e-03],\n",
              "       [6.92656636e-02, 1.31893798e-03, 1.38006953e-03, 2.30827820e-04,\n",
              "        1.82239525e-03, 1.01772207e-03],\n",
              "       [9.25309435e-02, 1.49874692e-03, 2.19344883e-03, 3.14618490e-04,\n",
              "        2.78495718e-03, 1.80815719e-03],\n",
              "       [9.58145559e-02, 1.37998408e-03, 1.36787992e-03, 2.52399128e-04,\n",
              "        2.98930681e-03, 8.84383859e-04],\n",
              "       [3.57755929e-01, 2.62918463e-03, 1.30957086e-02, 8.29293975e-04,\n",
              "        2.23135389e-02, 2.03730608e-03],\n",
              "       [1.73146054e-01, 1.49136246e-03, 2.70002172e-03, 3.24359164e-04,\n",
              "        5.40785631e-03, 1.35813525e-03],\n",
              "       [5.65175861e-02, 1.14315993e-03, 5.20399713e-04, 1.35110473e-04,\n",
              "        9.47293185e-04, 9.61452024e-04],\n",
              "       [2.67961591e-01, 2.03371723e-03, 5.34376269e-03, 6.85930427e-04,\n",
              "        1.14146098e-02, 1.25025585e-03],\n",
              "       [8.78079906e-02, 1.77643541e-03, 4.72980086e-03, 4.89630038e-04,\n",
              "        5.60235744e-03, 1.20808766e-03],\n",
              "       [4.82763082e-01, 2.37002736e-03, 1.19099431e-02, 5.91175107e-04,\n",
              "        1.85416210e-02, 1.31579174e-03],\n",
              "       [4.06398892e-01, 2.21195375e-03, 1.78701077e-02, 6.07297174e-04,\n",
              "        2.10052971e-02, 1.44055043e-03],\n",
              "       [7.88292140e-02, 1.89009006e-03, 4.48207790e-03, 2.35947664e-04,\n",
              "        5.64780738e-03, 1.47388875e-03],\n",
              "       [8.58360052e-01, 4.64024674e-03, 1.88276619e-01, 2.89048371e-03,\n",
              "        2.09482461e-01, 3.52690904e-03],\n",
              "       [1.23762041e-01, 1.61138491e-03, 1.45513972e-03, 2.23641182e-04,\n",
              "        2.75130523e-03, 1.50658539e-03],\n",
              "       [4.56397414e-01, 2.01629568e-03, 1.09870881e-02, 7.80495757e-04,\n",
              "        2.10161004e-02, 1.34309451e-03],\n",
              "       [1.73726350e-01, 1.63028797e-03, 2.68001435e-03, 3.52697563e-04,\n",
              "        4.79061157e-03, 1.08107342e-03],\n",
              "       [9.17707086e-02, 1.19869236e-03, 1.13048707e-03, 1.80252842e-04,\n",
              "        1.32305012e-03, 1.09010900e-03],\n",
              "       [2.39201207e-02, 1.02172897e-03, 3.30762938e-04, 7.58320530e-05,\n",
              "        6.65582076e-04, 1.12130574e-03],\n",
              "       [2.11247116e-01, 2.52942042e-03, 3.13359424e-02, 7.70760525e-04,\n",
              "        4.66786027e-02, 1.60128216e-03],\n",
              "       [1.81139022e-01, 1.68043212e-03, 2.51167384e-03, 2.71880504e-04,\n",
              "        2.80469470e-03, 1.23779592e-03],\n",
              "       [3.52202579e-02, 1.70210260e-03, 1.19029591e-03, 1.36868155e-04,\n",
              "        4.78191208e-03, 1.22333341e-03],\n",
              "       [8.16441059e-01, 6.94032013e-03, 3.18122327e-01, 1.88086815e-02,\n",
              "        6.58376753e-01, 4.92054923e-03],\n",
              "       [3.81284729e-02, 1.03608181e-03, 5.40025765e-04, 1.06376305e-04,\n",
              "        6.97016018e-04, 9.14303993e-04],\n",
              "       [1.59580827e-01, 1.40054966e-03, 2.49920762e-03, 2.84956943e-04,\n",
              "        5.66683104e-03, 1.14849291e-03],\n",
              "       [2.31033266e-02, 1.58101611e-03, 1.57261034e-03, 1.56077891e-04,\n",
              "        4.01591975e-03, 1.34770875e-03],\n",
              "       [1.07107185e-01, 1.38407550e-03, 1.66707824e-03, 2.33465049e-04,\n",
              "        2.91653187e-03, 1.43040263e-03],\n",
              "       [6.74124435e-02, 1.93443510e-03, 1.04985191e-02, 7.25809718e-04,\n",
              "        1.59356557e-02, 1.45586755e-03],\n",
              "       [1.67849332e-01, 1.72535982e-03, 1.83819979e-03, 2.68012256e-04,\n",
              "        2.30945880e-03, 1.67412753e-03],\n",
              "       [9.33801159e-02, 1.20584574e-03, 6.64647145e-04, 1.78025672e-04,\n",
              "        8.64347909e-04, 8.02361581e-04],\n",
              "       [5.35959043e-02, 1.26137328e-03, 9.36678087e-04, 1.20309589e-04,\n",
              "        1.31871249e-03, 9.57062817e-04],\n",
              "       [7.60353804e-01, 5.50474552e-03, 1.68189272e-01, 7.64584122e-03,\n",
              "        2.60910243e-01, 3.58136068e-03],\n",
              "       [4.12365682e-02, 1.18893792e-03, 3.71065486e-04, 8.94709156e-05,\n",
              "        5.62691130e-04, 1.17419858e-03],\n",
              "       [8.90405953e-01, 5.79064712e-03, 3.64027411e-01, 6.03397703e-03,\n",
              "        5.93359649e-01, 3.70553765e-03],\n",
              "       [1.33879408e-01, 1.75160856e-03, 2.43271748e-03, 3.99502664e-04,\n",
              "        4.22219886e-03, 2.21893773e-03],\n",
              "       [5.46346083e-02, 1.38227711e-03, 8.91186064e-04, 1.67773003e-04,\n",
              "        1.24898064e-03, 1.35665294e-03],\n",
              "       [2.74094731e-01, 1.32795714e-03, 3.05449846e-03, 3.36429744e-04,\n",
              "        4.86897631e-03, 9.40676313e-04],\n",
              "       [4.03702781e-02, 1.76299713e-03, 4.22773976e-03, 2.19596710e-04,\n",
              "        1.23641118e-02, 1.93585036e-03],\n",
              "       [3.76788266e-02, 1.15566968e-03, 7.66847981e-04, 1.37063267e-04,\n",
              "        2.13492149e-03, 1.18946517e-03],\n",
              "       [5.29387854e-02, 1.10882579e-03, 6.47041714e-04, 1.52216817e-04,\n",
              "        1.00594666e-03, 1.19578710e-03],\n",
              "       [7.35841841e-02, 1.65216520e-03, 3.18962988e-03, 2.12406623e-04,\n",
              "        4.04365454e-03, 1.41176337e-03],\n",
              "       [3.90121222e-01, 1.74518453e-03, 1.22294622e-02, 4.94202424e-04,\n",
              "        2.41279900e-02, 1.19913369e-03],\n",
              "       [1.94181323e-01, 1.67476188e-03, 2.62280018e-03, 3.71275702e-04,\n",
              "        4.05158196e-03, 1.44196895e-03],\n",
              "       [2.16810435e-01, 2.15300126e-03, 7.41234049e-03, 5.34816005e-04,\n",
              "        1.12595167e-02, 1.66027818e-03],\n",
              "       [2.04137504e-01, 1.67543907e-03, 5.23566455e-03, 4.20582801e-04,\n",
              "        8.00129492e-03, 1.65885850e-03],\n",
              "       [2.51821846e-01, 1.78058608e-03, 5.49145136e-03, 4.47784056e-04,\n",
              "        9.87206027e-03, 1.45150756e-03],\n",
              "       [1.53678954e-01, 1.79184030e-03, 3.90945561e-03, 5.16025524e-04,\n",
              "        8.44117999e-03, 1.63812132e-03],\n",
              "       [8.44095349e-01, 5.35590434e-03, 2.94090271e-01, 8.21267627e-03,\n",
              "        5.50132871e-01, 4.25359141e-03],\n",
              "       [9.03154969e-01, 5.30152488e-03, 4.43515450e-01, 1.11071561e-02,\n",
              "        7.24604726e-01, 4.63479245e-03],\n",
              "       [9.48896110e-02, 1.09767867e-03, 7.94206164e-04, 1.24896920e-04,\n",
              "        1.05661899e-03, 8.89863353e-04],\n",
              "       [3.07991710e-02, 1.71061663e-03, 3.23248911e-03, 2.54117622e-04,\n",
              "        9.55581572e-03, 1.64617272e-03],\n",
              "       [5.90167820e-01, 2.16720416e-03, 3.31191644e-02, 1.24169805e-03,\n",
              "        4.88940440e-02, 1.59472739e-03],\n",
              "       [8.96798670e-01, 5.63493604e-03, 4.58401740e-01, 1.01876594e-02,\n",
              "        7.14835823e-01, 4.41090064e-03],\n",
              "       [5.35358079e-02, 1.11612969e-03, 8.98122904e-04, 1.10635963e-04,\n",
              "        1.21057231e-03, 9.42396349e-04]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uVAWKam2ado5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "label_cols = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
        "\n",
        "test[\"toxic\"] = pred_probs[:,0]\n",
        "test[\"severe_toxic\"] = pred_probs[:,1]\n",
        "test[\"obscene\"] = pred_probs[:,2]\n",
        "test[\"threat\"] = pred_probs[:,3]\n",
        "test[\"insult\"] = pred_probs[:,4]\n",
        "test[\"identity_hate\"] = pred_probs[:,5]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r_l6LVlX_R9J",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 309
        },
        "outputId": "2b27935c-7832-4560-dd83-892d644e8aa4"
      },
      "source": [
        "test.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment_text</th>\n",
              "      <th>features</th>\n",
              "      <th>masks</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6699df82859122fa</th>\n",
              "      <td>JK kaylah we be frendsssssss and you dont suck...</td>\n",
              "      <td>[17, 1315, 267, 17, 1041, 117, 8637, 80, 39, 1...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.847499</td>\n",
              "      <td>0.005289</td>\n",
              "      <td>0.275254</td>\n",
              "      <td>0.011019</td>\n",
              "      <td>0.573681</td>\n",
              "      <td>0.003942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5e6f8e689bdfcede</th>\n",
              "      <td>But is a primarily clinical article.</td>\n",
              "      <td>[57, 27, 24, 3375, 4494, 1447, 9, 4, 3, 0, 0, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.080863</td>\n",
              "      <td>0.001501</td>\n",
              "      <td>0.001647</td>\n",
              "      <td>0.000293</td>\n",
              "      <td>0.002281</td>\n",
              "      <td>0.001326</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4b43be63e9cf076f</th>\n",
              "      <td>\":::::::Excuse me but it is not \"\"unilateral\"\"...</td>\n",
              "      <td>[17, 12, 60, 60, 60, 60, 60, 60, 60, 1506, 470...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.303586</td>\n",
              "      <td>0.001599</td>\n",
              "      <td>0.006832</td>\n",
              "      <td>0.000607</td>\n",
              "      <td>0.014862</td>\n",
              "      <td>0.001071</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58e26475c37b8425</th>\n",
              "      <td>*somewhat opposed; if mentioned in the lead, t...</td>\n",
              "      <td>[6940, 4202, 8337, 3168, 97, 108, 2911, 25, 18...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.211076</td>\n",
              "      <td>0.001794</td>\n",
              "      <td>0.004222</td>\n",
              "      <td>0.000510</td>\n",
              "      <td>0.009639</td>\n",
              "      <td>0.001515</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>e9d347116b62fd76</th>\n",
              "      <td>You what meters keeps making disruptive edits ...</td>\n",
              "      <td>[44, 113, 3556, 5788, 441, 27920, 8619, 23, 10...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.749881</td>\n",
              "      <td>0.004390</td>\n",
              "      <td>0.090431</td>\n",
              "      <td>0.001980</td>\n",
              "      <td>0.186765</td>\n",
              "      <td>0.002294</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                       comment_text  ... identity_hate\n",
              "id                                                                   ...              \n",
              "6699df82859122fa  JK kaylah we be frendsssssss and you dont suck...  ...      0.003942\n",
              "5e6f8e689bdfcede               But is a primarily clinical article.  ...      0.001326\n",
              "4b43be63e9cf076f  \":::::::Excuse me but it is not \"\"unilateral\"\"...  ...      0.001071\n",
              "58e26475c37b8425  *somewhat opposed; if mentioned in the lead, t...  ...      0.001515\n",
              "e9d347116b62fd76  You what meters keeps making disruptive edits ...  ...      0.002294\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dpE1pjPIAJGg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "4b99c155-6b04-4a37-9bfe-416093a7ab3e"
      },
      "source": [
        "test_to_csv = test.reset_index()\n",
        "test_to_csv.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>features</th>\n",
              "      <th>masks</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>6699df82859122fa</td>\n",
              "      <td>JK kaylah we be frendsssssss and you dont suck...</td>\n",
              "      <td>[17, 1315, 267, 17, 1041, 117, 8637, 80, 39, 1...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.847499</td>\n",
              "      <td>0.005289</td>\n",
              "      <td>0.275254</td>\n",
              "      <td>0.011019</td>\n",
              "      <td>0.573681</td>\n",
              "      <td>0.003942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>5e6f8e689bdfcede</td>\n",
              "      <td>But is a primarily clinical article.</td>\n",
              "      <td>[57, 27, 24, 3375, 4494, 1447, 9, 4, 3, 0, 0, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.080863</td>\n",
              "      <td>0.001501</td>\n",
              "      <td>0.001647</td>\n",
              "      <td>0.000293</td>\n",
              "      <td>0.002281</td>\n",
              "      <td>0.001326</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4b43be63e9cf076f</td>\n",
              "      <td>\":::::::Excuse me but it is not \"\"unilateral\"\"...</td>\n",
              "      <td>[17, 12, 60, 60, 60, 60, 60, 60, 60, 1506, 470...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.303586</td>\n",
              "      <td>0.001599</td>\n",
              "      <td>0.006832</td>\n",
              "      <td>0.000607</td>\n",
              "      <td>0.014862</td>\n",
              "      <td>0.001071</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>58e26475c37b8425</td>\n",
              "      <td>*somewhat opposed; if mentioned in the lead, t...</td>\n",
              "      <td>[6940, 4202, 8337, 3168, 97, 108, 2911, 25, 18...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.211076</td>\n",
              "      <td>0.001794</td>\n",
              "      <td>0.004222</td>\n",
              "      <td>0.000510</td>\n",
              "      <td>0.009639</td>\n",
              "      <td>0.001515</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>e9d347116b62fd76</td>\n",
              "      <td>You what meters keeps making disruptive edits ...</td>\n",
              "      <td>[44, 113, 3556, 5788, 441, 27920, 8619, 23, 10...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.749881</td>\n",
              "      <td>0.004390</td>\n",
              "      <td>0.090431</td>\n",
              "      <td>0.001980</td>\n",
              "      <td>0.186765</td>\n",
              "      <td>0.002294</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id  ... identity_hate\n",
              "0  6699df82859122fa  ...      0.003942\n",
              "1  5e6f8e689bdfcede  ...      0.001326\n",
              "2  4b43be63e9cf076f  ...      0.001071\n",
              "3  58e26475c37b8425  ...      0.001515\n",
              "4  e9d347116b62fd76  ...      0.002294\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mF95PeFM_rDh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cwd = os.getcwd()\n",
        "pred_save_path = output_model_file = os.path.join(cwd, \"drive/My Drive/Data/toxic_1_epoch.csv\")\n",
        "test_to_csv[[\"id\", \"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]].to_csv(pred_save_path, index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tBnRrhxrjlEj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "32e16c7b-ce56-49ce-cf30-e729eb2010fa"
      },
      "source": [
        "np.round(pred_probs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1., 0., 0., 0., 1., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 1., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 1., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 1., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 1., 0.],\n",
              "       [1., 0., 0., 0., 1., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 1., 0.],\n",
              "       [0., 0., 0., 0., 0., 0.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yHv4x0ZqJC1s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "eb54339f-ae21-4de0-b41c-5764c4e02273"
      },
      "source": [
        "from google.colab import files\n",
        "\n",
        "test.to_csv('test_sample.csv')\n",
        "files.download('test_sample.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_c8d1b073-d164-4dd6-b4cb-26aa1209268a\", \"test_sample.csv\", 260458)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}