xlnet_multi_class_chinese_english.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "xlnet-multi-class-chinese-english.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "12ec24c4f7fd4ac289a9449d6635d822": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_932286b9f88945da8195beb37d958d28",
              "IPY_MODEL_925d7e995e0e417083621354b8b7a581",
              "IPY_MODEL_deeddff422b94d4d8612657b31b3fa78"
            ],
            "layout": "IPY_MODEL_383ccc73cfc3457da8c7d425c80163f3"
          }
        },
        "932286b9f88945da8195beb37d958d28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_97614babd4a6407c81fcaad156a6c994",
            "placeholder": "​",
            "style": "IPY_MODEL_1326c9600f0d4f94921a53fe8dd2ad73",
            "value": "Downloading: 100%"
          }
        },
        "925d7e995e0e417083621354b8b7a581": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d626d1c57cd64641966a97237c25a515",
            "max": 19,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_01f6bb96fa6743a69905528275cd69fd",
            "value": 19
          }
        },
        "deeddff422b94d4d8612657b31b3fa78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a5ad828c7af4f3595b892f033bec2fa",
            "placeholder": "​",
            "style": "IPY_MODEL_d60085b995ef402f99efb1e8f7336649",
            "value": " 19.0/19.0 [00:00&lt;00:00, 414B/s]"
          }
        },
        "383ccc73cfc3457da8c7d425c80163f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "97614babd4a6407c81fcaad156a6c994": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1326c9600f0d4f94921a53fe8dd2ad73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d626d1c57cd64641966a97237c25a515": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01f6bb96fa6743a69905528275cd69fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2a5ad828c7af4f3595b892f033bec2fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d60085b995ef402f99efb1e8f7336649": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "383d55ed3cce4256a6c0e7737dc7b630": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a0f65a084ff94e65a05dc10bc3ae52e5",
              "IPY_MODEL_fe121353df7944429d7c201abe81e166",
              "IPY_MODEL_4aea50680de04df8a298996b4eeb7b1b"
            ],
            "layout": "IPY_MODEL_a13aa4d12a0147df83bd432e4fdaa336"
          }
        },
        "a0f65a084ff94e65a05dc10bc3ae52e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e0a9179eaa004340a237f735011ec888",
            "placeholder": "​",
            "style": "IPY_MODEL_2e13ffe4de19454897e7051977cbb02e",
            "value": "Downloading: 100%"
          }
        },
        "fe121353df7944429d7c201abe81e166": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d2359a7725f54d8fa4afa46e5c76ec9e",
            "max": 671,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_14356b3213a9453ba8d65c6cb69d824b",
            "value": 671
          }
        },
        "4aea50680de04df8a298996b4eeb7b1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_de8688d599b0449ba5626536da46c7a7",
            "placeholder": "​",
            "style": "IPY_MODEL_1eee3b51f6d740cebc91d1c9f91da360",
            "value": " 671/671 [00:00&lt;00:00, 13.9kB/s]"
          }
        },
        "a13aa4d12a0147df83bd432e4fdaa336": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0a9179eaa004340a237f735011ec888": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e13ffe4de19454897e7051977cbb02e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d2359a7725f54d8fa4afa46e5c76ec9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "14356b3213a9453ba8d65c6cb69d824b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "de8688d599b0449ba5626536da46c7a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1eee3b51f6d740cebc91d1c9f91da360": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "754575e2661f4c46ba278bfc1b495898": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5135776b7a0640c9afd028c7931f068b",
              "IPY_MODEL_eef727486fe94cb6b51194b1d0717d5a",
              "IPY_MODEL_c0f9230635d148f3b0020722aededa0c"
            ],
            "layout": "IPY_MODEL_afb6567472fd474a86f27ed785d18453"
          }
        },
        "5135776b7a0640c9afd028c7931f068b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_86d58a009f994101916ce36fa6f3c2be",
            "placeholder": "​",
            "style": "IPY_MODEL_6ebef002e08f4ab6aef300805ecb09fe",
            "value": "Downloading: 100%"
          }
        },
        "eef727486fe94cb6b51194b1d0717d5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b60c790e1fb4ca7afb80f7e6b85df1a",
            "max": 691427,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_73e6c3c9dc76461c83fcae3b72d2e7d0",
            "value": 691427
          }
        },
        "c0f9230635d148f3b0020722aededa0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a0237c505b04a38bb8573b92651f2c9",
            "placeholder": "​",
            "style": "IPY_MODEL_1ae09c2304264ba18e56fdd12b656a30",
            "value": " 675k/675k [00:00&lt;00:00, 1.22MB/s]"
          }
        },
        "afb6567472fd474a86f27ed785d18453": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86d58a009f994101916ce36fa6f3c2be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ebef002e08f4ab6aef300805ecb09fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b60c790e1fb4ca7afb80f7e6b85df1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73e6c3c9dc76461c83fcae3b72d2e7d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8a0237c505b04a38bb8573b92651f2c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1ae09c2304264ba18e56fdd12b656a30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bbaa4c6d8ad8438c95575a25d102a936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cba48eb6faca4b99a3523c85cb6feeed",
              "IPY_MODEL_398f893824cf46189526e60d8256c0d9",
              "IPY_MODEL_995ba8b5054d4074957a4ad6961d4c2f"
            ],
            "layout": "IPY_MODEL_87784e6e5a3f407f86708205d78e63cd"
          }
        },
        "cba48eb6faca4b99a3523c85cb6feeed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ba4797138be46eabbfcf387fa442b8f",
            "placeholder": "​",
            "style": "IPY_MODEL_f70c9fea9169431686f6e0c10f6305f2",
            "value": "Downloading: 100%"
          }
        },
        "398f893824cf46189526e60d8256c0d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78988bbcd40a4e279487c4b16a17ba12",
            "max": 1273988,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9b97123b79034db4bcabc1ae8b0ec4d8",
            "value": 1273988
          }
        },
        "995ba8b5054d4074957a4ad6961d4c2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e425e16687374d91ae466640ef565ee0",
            "placeholder": "​",
            "style": "IPY_MODEL_c331e9d751384f918967178c3d5ff359",
            "value": " 1.21M/1.21M [00:00&lt;00:00, 1.03MB/s]"
          }
        },
        "87784e6e5a3f407f86708205d78e63cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ba4797138be46eabbfcf387fa442b8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f70c9fea9169431686f6e0c10f6305f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "78988bbcd40a4e279487c4b16a17ba12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b97123b79034db4bcabc1ae8b0ec4d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e425e16687374d91ae466640ef565ee0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c331e9d751384f918967178c3d5ff359": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fe7d4980f3d04ecaa84f480b9cf848ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7892ee95d46f43958c7edffafb8f91f6",
              "IPY_MODEL_c6ff0e0bacad41c186faa84232c36292",
              "IPY_MODEL_7cca2316e2054ee5a17b9ed34c1b0663"
            ],
            "layout": "IPY_MODEL_1a6ff00d515c4934b36bc0bbc795a449"
          }
        },
        "7892ee95d46f43958c7edffafb8f91f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7e71022590c04a7c9f1b1fbb298b88e6",
            "placeholder": "​",
            "style": "IPY_MODEL_36f3a13fb1da439ba09bf4cb8eebfbe9",
            "value": "Downloading: 100%"
          }
        },
        "c6ff0e0bacad41c186faa84232c36292": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_63b9db3b37e342278bd4c506170e0b81",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bd9ffdd5226a4a15a546f8123c86f39d",
            "value": 2
          }
        },
        "7cca2316e2054ee5a17b9ed34c1b0663": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1906049cca034cedbfe34d56159b7af3",
            "placeholder": "​",
            "style": "IPY_MODEL_60e522fe0da84cbbb56f56666c37249b",
            "value": " 2.00/2.00 [00:00&lt;00:00, 47.9B/s]"
          }
        },
        "1a6ff00d515c4934b36bc0bbc795a449": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e71022590c04a7c9f1b1fbb298b88e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "36f3a13fb1da439ba09bf4cb8eebfbe9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "63b9db3b37e342278bd4c506170e0b81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd9ffdd5226a4a15a546f8123c86f39d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1906049cca034cedbfe34d56159b7af3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60e522fe0da84cbbb56f56666c37249b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "91d4796259bd43a8b1ab73d21ae051f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7244cc74f9b94ee1b8b07f8fac51efd9",
              "IPY_MODEL_978412ffa319434084032ac45b1cd618",
              "IPY_MODEL_72e37476f28143deb76ca56cf514d379"
            ],
            "layout": "IPY_MODEL_a905d9b3364543b1b2eb155cea5f4105"
          }
        },
        "7244cc74f9b94ee1b8b07f8fac51efd9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6a64f0a5fd045e098d3f011c8346244",
            "placeholder": "​",
            "style": "IPY_MODEL_f87c5e4134e648d5812ea15e62029351",
            "value": "Downloading: 100%"
          }
        },
        "978412ffa319434084032ac45b1cd618": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_375bdfc699154cabab93d4c4317227b5",
            "max": 202,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2039d004198d4d759f68e13eccc5aef0",
            "value": 202
          }
        },
        "72e37476f28143deb76ca56cf514d379": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_243a534249aa4a22a93c5ef70c289db7",
            "placeholder": "​",
            "style": "IPY_MODEL_1fbee335b823479192406c1b911c4cfb",
            "value": " 202/202 [00:00&lt;00:00, 5.62kB/s]"
          }
        },
        "a905d9b3364543b1b2eb155cea5f4105": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f6a64f0a5fd045e098d3f011c8346244": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f87c5e4134e648d5812ea15e62029351": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "375bdfc699154cabab93d4c4317227b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2039d004198d4d759f68e13eccc5aef0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "243a534249aa4a22a93c5ef70c289db7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1fbee335b823479192406c1b911c4cfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dd941141a74240a0b511215e71ae98d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9e1a945e631c49edbdfd4b69af8cbef0",
              "IPY_MODEL_b3da2ad6362d49c6971f450b65c9b672",
              "IPY_MODEL_de52d1d0781a4e028a4119ef5915d353"
            ],
            "layout": "IPY_MODEL_ecc6276356734878bd2480e18da25587"
          }
        },
        "9e1a945e631c49edbdfd4b69af8cbef0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6bf46a63168f429a879848946dc97c41",
            "placeholder": "​",
            "style": "IPY_MODEL_25c35d71e96240a18cb7e02189ef9bd5",
            "value": "Downloading: 100%"
          }
        },
        "b3da2ad6362d49c6971f450b65c9b672": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_496c2521f5304feda41ba86528a88537",
            "max": 467042048,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cd7bea6a11054cf5a19b952a0b23fa46",
            "value": 467042048
          }
        },
        "de52d1d0781a4e028a4119ef5915d353": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2d7f2d4b116b41549dd13744888f571a",
            "placeholder": "​",
            "style": "IPY_MODEL_c0b8ee64545748b49e9616028f0187e2",
            "value": " 445M/445M [00:13&lt;00:00, 37.4MB/s]"
          }
        },
        "ecc6276356734878bd2480e18da25587": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6bf46a63168f429a879848946dc97c41": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25c35d71e96240a18cb7e02189ef9bd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "496c2521f5304feda41ba86528a88537": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd7bea6a11054cf5a19b952a0b23fa46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2d7f2d4b116b41549dd13744888f571a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c0b8ee64545748b49e9616028f0187e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/angel870326/XLNet_Text_Classification/blob/main/xlnet_multi_class_chinese_english.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "> 2022.05.14 Ssu-Yun Wang<br/>\n",
        "[Github @angel870326](https://github.com/angel870326)\n",
        "\n",
        "# **XLNet Multi-class Text Classification for Chinese and English**\n",
        "Available for both multi-lable and single-label.\n",
        "\n",
        "## **Contents**\n",
        "1. Dataset\n",
        "   *   Raw dataset\n",
        "   *   Clean dataset\n",
        "   *   Convert 'product' to dummy variables\n",
        "   *   Get small sample of train and test to try (optional)\n",
        "\n",
        "2. Preprocessing<br/>\n",
        "   (1) XLNetTokenizer & Sentence embedding<br/>\n",
        "   (2) Tokenize input texts into ids<br/>\n",
        "   (3) Create attention masks<br/>\n",
        "   (4) Add input ids and attention masks to the dataframe<br/>\n",
        "3. Train, Valid Split\n",
        "4. Create Dataloaders (torch tensors)\n",
        "5. Training<br/>\n",
        "   (1) Functions<br/>\n",
        "   (2) Train model from scratch<br/>\n",
        "   (3) Train model from previous checkpoint<br/>\n",
        "6. Get Predictions<br/>\n",
        "   (1) Testing<br/>\n",
        "   (2) For single-label classification<br/>\n",
        "   (3) Save the predicting results<br/>\n",
        "\n",
        "\n",
        "** Modified from  https://towardsdatascience.com/multi-label-text-classification-with-xlnet-b5f5755302df*"
      ],
      "metadata": {
        "id": "8t0uK6uS9pMP"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aUzQGX2ILqMK",
        "outputId": "0d5dc7d9-b15b-47eb-f376-4834c1ba4e41",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install transformers\n",
        "!pip install sentencepiece"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.19.1-py3-none-any.whl (4.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.2 MB 5.3 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.6.0-py3-none-any.whl (84 kB)\n",
            "\u001b[K     |████████████████████████████████| 84 kB 3.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 46.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 48.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Installing collected packages: pyyaml, tokenizers, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.6.0 pyyaml-6.0 tokenizers-0.12.1 transformers-4.19.1\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 60 kB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.96\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NwTHAUqXk7_B"
      },
      "source": [
        "import os\n",
        "import math\n",
        "\n",
        "import torch\n",
        "from torch.nn import BCEWithLogitsLoss\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import AdamW, XLNetTokenizer, XLNetModel, XLNetLMHeadModel, XLNetConfig\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm, trange\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o4ZY1PcIlfKW",
        "outputId": "f44fa63d-7bd6-4c60-fafa-af71dc386c44",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# GPU\n",
        "print(\"GPU Available: {}\".format(torch.cuda.is_available()))\n",
        "n_gpu = torch.cuda.device_count()\n",
        "print(\"Number of GPU Available: {}\".format(n_gpu))\n",
        "print(\"GPU: {}\".format(torch.cuda.get_device_name(0)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPU Available: True\n",
            "Number of GPU Available: 1\n",
            "GPU: Tesla K80\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vothBTEKll78"
      },
      "source": [
        "# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FZbLvzAKlmsP"
      },
      "source": [
        "## **1. Dataset**\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IlQpf1MmlqJI",
        "outputId": "f40dc760-b9d4-4189-9dab-104cda0fac19",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Raw dataset"
      ],
      "metadata": {
        "id": "mHwlqEPQIvhK"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a2Xf8DAOl0XH"
      },
      "source": [
        "train_raw = pd.read_csv('/content/drive/MyDrive/碩一下/金融科技/麻布數據/資料檔/品類_clean.csv')\n",
        "test_raw = pd.read_csv('/content/drive/MyDrive/碩一下/金融科技/麻布數據/資料檔/原始資料/cat_test_v2(question).csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1CjLi8LdP1b",
        "outputId": "d019e91d-47f5-4a22-ef0b-42f790b5c1a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "train_raw.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0                     name product       clean_name\n",
              "0           0  r視舒坦單支裝人工淚液點眼液0.5ml24支入    人工淚液  r視舒坦單支裝人工淚液點眼液入\n",
              "1           1               優麗舒點眼液15ml    人工淚液           優麗舒點眼液\n",
              "2           2              (r)視舒坦單支裝人工    人工淚液        r視舒坦單支裝人工\n",
              "3           3                rohto乾眼淚液    人工淚液        rohto乾眼淚液\n",
              "4           4        r參天柔潤人工淚液點眼液5ml4入    人工淚液     r參天柔潤人工淚液點眼液"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-112c3201-6cdd-4d82-8299-4b715a77a215\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>name</th>\n",
              "      <th>product</th>\n",
              "      <th>clean_name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>r視舒坦單支裝人工淚液點眼液0.5ml24支入</td>\n",
              "      <td>人工淚液</td>\n",
              "      <td>r視舒坦單支裝人工淚液點眼液入</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>優麗舒點眼液15ml</td>\n",
              "      <td>人工淚液</td>\n",
              "      <td>優麗舒點眼液</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>(r)視舒坦單支裝人工</td>\n",
              "      <td>人工淚液</td>\n",
              "      <td>r視舒坦單支裝人工</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>rohto乾眼淚液</td>\n",
              "      <td>人工淚液</td>\n",
              "      <td>rohto乾眼淚液</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>r參天柔潤人工淚液點眼液5ml4入</td>\n",
              "      <td>人工淚液</td>\n",
              "      <td>r參天柔潤人工淚液點眼液</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-112c3201-6cdd-4d82-8299-4b715a77a215')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-112c3201-6cdd-4d82-8299-4b715a77a215 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-112c3201-6cdd-4d82-8299-4b715a77a215');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QWX6qJg-dRcR",
        "outputId": "12020105-bad2-4f19-c6c9-7bfbe9133c5f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        }
      },
      "source": [
        "test_raw.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                  name occurred_at  iv_price  product  units  \\\n",
              "0                     【施巴】嬌顏卸妝乳HA400ml  2022-01-08       968      NaN    1.0   \n",
              "1                         艾瑪絲舒敏平衡洗髮精40  2022-01-02      3054      NaN    2.0   \n",
              "2   【BB LOVE】高質感天絲TENCEL舖棉兩用兒童睡袋(多款可選)  2022-01-02       586      NaN    1.0   \n",
              "3        Lovita愛維他-緩釋型維生素C 1000mg(60錠)  2022-01-01      2180      NaN    1.0   \n",
              "4  111262674618 CM 12公分長效Rifle風扇 1200轉  2022-01-07       129      NaN    1.0   \n",
              "\n",
              "   unit_price  total_price                           channel  \n",
              "0       705.0        705.0  bceb22fe7ebb8d831758f603636b37ec  \n",
              "1       488.0        976.0  5a196ac44d260b2e4abcc5e8f863c0e5  \n",
              "2       990.0        990.0  bceb22fe7ebb8d831758f603636b37ec  \n",
              "3       440.0        440.0  4a3d67cc0cd76bf56f18b40ed671bb05  \n",
              "4       129.0        129.0  fb92a6a55cb514d720672e54dbe11f15  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5e50b3ae-1109-4129-8782-409c6ddee2f0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>occurred_at</th>\n",
              "      <th>iv_price</th>\n",
              "      <th>product</th>\n",
              "      <th>units</th>\n",
              "      <th>unit_price</th>\n",
              "      <th>total_price</th>\n",
              "      <th>channel</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>【施巴】嬌顏卸妝乳HA400ml</td>\n",
              "      <td>2022-01-08</td>\n",
              "      <td>968</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "      <td>705.0</td>\n",
              "      <td>705.0</td>\n",
              "      <td>bceb22fe7ebb8d831758f603636b37ec</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>艾瑪絲舒敏平衡洗髮精40</td>\n",
              "      <td>2022-01-02</td>\n",
              "      <td>3054</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2.0</td>\n",
              "      <td>488.0</td>\n",
              "      <td>976.0</td>\n",
              "      <td>5a196ac44d260b2e4abcc5e8f863c0e5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>【BB LOVE】高質感天絲TENCEL舖棉兩用兒童睡袋(多款可選)</td>\n",
              "      <td>2022-01-02</td>\n",
              "      <td>586</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "      <td>990.0</td>\n",
              "      <td>990.0</td>\n",
              "      <td>bceb22fe7ebb8d831758f603636b37ec</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Lovita愛維他-緩釋型維生素C 1000mg(60錠)</td>\n",
              "      <td>2022-01-01</td>\n",
              "      <td>2180</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "      <td>440.0</td>\n",
              "      <td>440.0</td>\n",
              "      <td>4a3d67cc0cd76bf56f18b40ed671bb05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>111262674618 CM 12公分長效Rifle風扇 1200轉</td>\n",
              "      <td>2022-01-07</td>\n",
              "      <td>129</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "      <td>129.0</td>\n",
              "      <td>129.0</td>\n",
              "      <td>fb92a6a55cb514d720672e54dbe11f15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5e50b3ae-1109-4129-8782-409c6ddee2f0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5e50b3ae-1109-4129-8782-409c6ddee2f0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5e50b3ae-1109-4129-8782-409c6ddee2f0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Clean dataset"
      ],
      "metadata": {
        "id": "7BHK5S2EI2De"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zz9OzE2TmSN1",
        "outputId": "9ff67766-25f2-4269-b268-debb1726457b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        }
      },
      "source": [
        "train_data = {'id': range(1, len(train_raw) + 1),\n",
        "              'name': train_raw['clean_name'],\n",
        "              'product': train_raw['product']}\n",
        "train_clean = pd.DataFrame(train_data)\n",
        "train_clean"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          id                   name product\n",
              "0          1        r視舒坦單支裝人工淚液點眼液入    人工淚液\n",
              "1          2                 優麗舒點眼液    人工淚液\n",
              "2          3              r視舒坦單支裝人工    人工淚液\n",
              "3          4              rohto乾眼淚液    人工淚液\n",
              "4          5           r參天柔潤人工淚液點眼液    人工淚液\n",
              "...      ...                    ...     ...\n",
              "91914  91915   mdmmd微醺心機漸層腮紅-玫瑰瑪格麗特      腮紅\n",
              "91915  91916       mdmmd微醺心機漸層腮紅-橙酒      腮紅\n",
              "91916  91917     倫敦芮魅叛逆玩色腮紅盤005微醺酒紅      腮紅\n",
              "91917  91918   palladio柔光幻彩貝殼腮紅微醺蜜桃      腮紅\n",
              "91918  91919  韓國e-glips粉嫩微醺美肌花瓣腮紅可選      腮紅\n",
              "\n",
              "[91919 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1ef23108-bc04-49db-9278-3fcb2e7f84ad\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "      <th>product</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>r視舒坦單支裝人工淚液點眼液入</td>\n",
              "      <td>人工淚液</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>優麗舒點眼液</td>\n",
              "      <td>人工淚液</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>r視舒坦單支裝人工</td>\n",
              "      <td>人工淚液</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>rohto乾眼淚液</td>\n",
              "      <td>人工淚液</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>r參天柔潤人工淚液點眼液</td>\n",
              "      <td>人工淚液</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91914</th>\n",
              "      <td>91915</td>\n",
              "      <td>mdmmd微醺心機漸層腮紅-玫瑰瑪格麗特</td>\n",
              "      <td>腮紅</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91915</th>\n",
              "      <td>91916</td>\n",
              "      <td>mdmmd微醺心機漸層腮紅-橙酒</td>\n",
              "      <td>腮紅</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91916</th>\n",
              "      <td>91917</td>\n",
              "      <td>倫敦芮魅叛逆玩色腮紅盤005微醺酒紅</td>\n",
              "      <td>腮紅</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91917</th>\n",
              "      <td>91918</td>\n",
              "      <td>palladio柔光幻彩貝殼腮紅微醺蜜桃</td>\n",
              "      <td>腮紅</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91918</th>\n",
              "      <td>91919</td>\n",
              "      <td>韓國e-glips粉嫩微醺美肌花瓣腮紅可選</td>\n",
              "      <td>腮紅</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>91919 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1ef23108-bc04-49db-9278-3fcb2e7f84ad')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1ef23108-bc04-49db-9278-3fcb2e7f84ad button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1ef23108-bc04-49db-9278-3fcb2e7f84ad');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oW3dJFOTMkq1",
        "outputId": "93a2ec05-e9b1-48da-b725-f41a63acbdf3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        }
      },
      "source": [
        "test_data = {'id': range(1, len(test_raw) + 1),\n",
        "              'name': test_raw['name']}\n",
        "test = pd.DataFrame(test_data)\n",
        "test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            id                                    name\n",
              "0            1                        【施巴】嬌顏卸妝乳HA400ml\n",
              "1            2                            艾瑪絲舒敏平衡洗髮精40\n",
              "2            3      【BB LOVE】高質感天絲TENCEL舖棉兩用兒童睡袋(多款可選)\n",
              "3            4           Lovita愛維他-緩釋型維生素C 1000mg(60錠)\n",
              "4            5     111262674618 CM 12公分長效Rifle風扇 1200轉\n",
              "...        ...                                     ...\n",
              "108846  108847                         歐舒丹~橙花蘭花護手霜30ml\n",
              "108847  108848                      高露潔 360度抗敏專家牙刷(1入)\n",
              "108848  108849                高露潔專業型超小刷頭密植軟毛牙刷6支裝【康是美】\n",
              "108849  108850  熊寶貝 沁藍海洋香柔軟衣物護衣精2+6組 (3.2Lx2+1.84Lx6包)\n",
              "108850  108851                               金蘭油膏295ml\n",
              "\n",
              "[108851 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2a81f218-31e1-4fc4-89b1-daf5997a0e01\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>【施巴】嬌顏卸妝乳HA400ml</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>艾瑪絲舒敏平衡洗髮精40</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>【BB LOVE】高質感天絲TENCEL舖棉兩用兒童睡袋(多款可選)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>Lovita愛維他-緩釋型維生素C 1000mg(60錠)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>111262674618 CM 12公分長效Rifle風扇 1200轉</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108846</th>\n",
              "      <td>108847</td>\n",
              "      <td>歐舒丹~橙花蘭花護手霜30ml</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108847</th>\n",
              "      <td>108848</td>\n",
              "      <td>高露潔 360度抗敏專家牙刷(1入)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108848</th>\n",
              "      <td>108849</td>\n",
              "      <td>高露潔專業型超小刷頭密植軟毛牙刷6支裝【康是美】</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108849</th>\n",
              "      <td>108850</td>\n",
              "      <td>熊寶貝 沁藍海洋香柔軟衣物護衣精2+6組 (3.2Lx2+1.84Lx6包)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108850</th>\n",
              "      <td>108851</td>\n",
              "      <td>金蘭油膏295ml</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>108851 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2a81f218-31e1-4fc4-89b1-daf5997a0e01')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2a81f218-31e1-4fc4-89b1-daf5997a0e01 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2a81f218-31e1-4fc4-89b1-daf5997a0e01');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Convert 'product' to dummy variables"
      ],
      "metadata": {
        "id": "fYHYGiUSXixZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train = train_clean[['id', 'name']]\n",
        "products = pd.get_dummies(train_clean['product'])\n",
        "train = pd.concat([train, products], axis=1)\n",
        "train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 589
        },
        "id": "WYbewHhYXlR_",
        "outputId": "6e19f9e6-035e-44fd-dcfd-dd632f70d7b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          id                   name  bb霜  cc霜  dd霜  rtd調酒  中式香腸  乳酸菌  人工淚液  \\\n",
              "0          1        r視舒坦單支裝人工淚液點眼液入    0    0    0      0     0    0     1   \n",
              "1          2                 優麗舒點眼液    0    0    0      0     0    0     1   \n",
              "2          3              r視舒坦單支裝人工    0    0    0      0     0    0     1   \n",
              "3          4              rohto乾眼淚液    0    0    0      0     0    0     1   \n",
              "4          5           r參天柔潤人工淚液點眼液    0    0    0      0     0    0     1   \n",
              "...      ...                    ...  ...  ...  ...    ...   ...  ...   ...   \n",
              "91914  91915   mdmmd微醺心機漸層腮紅-玫瑰瑪格麗特    0    0    0      0     0    0     0   \n",
              "91915  91916       mdmmd微醺心機漸層腮紅-橙酒    0    0    0      0     0    0     0   \n",
              "91916  91917     倫敦芮魅叛逆玩色腮紅盤005微醺酒紅    0    0    0      0     0    0     0   \n",
              "91917  91918   palladio柔光幻彩貝殼腮紅微醺蜜桃    0    0    0      0     0    0     0   \n",
              "91918  91919  韓國e-glips粉嫩微醺美肌花瓣腮紅可選    0    0    0      0     0    0     0   \n",
              "\n",
              "       保久乳  ...  電磁爐  電視遊戲機  電鍋  電風扇  面膜  面霜/乳霜  香氛機  香氛蠟燭  鮮乳  麥片穀類  \n",
              "0        0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "1        0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "2        0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "3        0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "4        0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "...    ...  ...  ...    ...  ..  ...  ..    ...  ...   ...  ..   ...  \n",
              "91914    0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "91915    0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "91916    0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "91917    0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "91918    0  ...    0      0   0    0   0      0    0     0   0     0  \n",
              "\n",
              "[91919 rows x 218 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-de158187-e01c-455a-9ab5-eb09b0277728\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "      <th>bb霜</th>\n",
              "      <th>cc霜</th>\n",
              "      <th>dd霜</th>\n",
              "      <th>rtd調酒</th>\n",
              "      <th>中式香腸</th>\n",
              "      <th>乳酸菌</th>\n",
              "      <th>人工淚液</th>\n",
              "      <th>保久乳</th>\n",
              "      <th>...</th>\n",
              "      <th>電磁爐</th>\n",
              "      <th>電視遊戲機</th>\n",
              "      <th>電鍋</th>\n",
              "      <th>電風扇</th>\n",
              "      <th>面膜</th>\n",
              "      <th>面霜/乳霜</th>\n",
              "      <th>香氛機</th>\n",
              "      <th>香氛蠟燭</th>\n",
              "      <th>鮮乳</th>\n",
              "      <th>麥片穀類</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>r視舒坦單支裝人工淚液點眼液入</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>優麗舒點眼液</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>r視舒坦單支裝人工</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>rohto乾眼淚液</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>r參天柔潤人工淚液點眼液</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91914</th>\n",
              "      <td>91915</td>\n",
              "      <td>mdmmd微醺心機漸層腮紅-玫瑰瑪格麗特</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91915</th>\n",
              "      <td>91916</td>\n",
              "      <td>mdmmd微醺心機漸層腮紅-橙酒</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91916</th>\n",
              "      <td>91917</td>\n",
              "      <td>倫敦芮魅叛逆玩色腮紅盤005微醺酒紅</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91917</th>\n",
              "      <td>91918</td>\n",
              "      <td>palladio柔光幻彩貝殼腮紅微醺蜜桃</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91918</th>\n",
              "      <td>91919</td>\n",
              "      <td>韓國e-glips粉嫩微醺美肌花瓣腮紅可選</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>91919 rows × 218 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-de158187-e01c-455a-9ab5-eb09b0277728')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-de158187-e01c-455a-9ab5-eb09b0277728 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-de158187-e01c-455a-9ab5-eb09b0277728');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Get small sample of train and test to try (optional)"
      ],
      "metadata": {
        "id": "C9nIBthXJW8Y"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ZEGGs69tD-3"
      },
      "source": [
        "train = train.sample(n=1000)\n",
        "test = train.sample(n=10)[['id','name']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MN8bQOt2mVc5"
      },
      "source": [
        "## **2. Preprocessing**\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### (1) XLNetTokenizer & Sentence embedding"
      ],
      "metadata": {
        "id": "v8gloWkVKSrE"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cz44ag9VMutg"
      },
      "source": [
        "def plot_sentence_embeddings_length(text_list, tokenizer):\n",
        "    tokenized_texts = list(map(lambda t: tokenizer.tokenize(t), text_list))\n",
        "    tokenized_texts_len = list(map(lambda t: len(t), tokenized_texts))\n",
        "    fig, ax = plt.subplots(figsize=(8, 5));\n",
        "    ax.hist(tokenized_texts_len, bins=40);\n",
        "    ax.set_xlabel(\"Length of Comment Embeddings\");\n",
        "    ax.set_ylabel(\"Number of Comments\");\n",
        "    return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PTOd-qKQMyFO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 209,
          "referenced_widgets": [
            "12ec24c4f7fd4ac289a9449d6635d822",
            "932286b9f88945da8195beb37d958d28",
            "925d7e995e0e417083621354b8b7a581",
            "deeddff422b94d4d8612657b31b3fa78",
            "383ccc73cfc3457da8c7d425c80163f3",
            "97614babd4a6407c81fcaad156a6c994",
            "1326c9600f0d4f94921a53fe8dd2ad73",
            "d626d1c57cd64641966a97237c25a515",
            "01f6bb96fa6743a69905528275cd69fd",
            "2a5ad828c7af4f3595b892f033bec2fa",
            "d60085b995ef402f99efb1e8f7336649",
            "383d55ed3cce4256a6c0e7737dc7b630",
            "a0f65a084ff94e65a05dc10bc3ae52e5",
            "fe121353df7944429d7c201abe81e166",
            "4aea50680de04df8a298996b4eeb7b1b",
            "a13aa4d12a0147df83bd432e4fdaa336",
            "e0a9179eaa004340a237f735011ec888",
            "2e13ffe4de19454897e7051977cbb02e",
            "d2359a7725f54d8fa4afa46e5c76ec9e",
            "14356b3213a9453ba8d65c6cb69d824b",
            "de8688d599b0449ba5626536da46c7a7",
            "1eee3b51f6d740cebc91d1c9f91da360",
            "754575e2661f4c46ba278bfc1b495898",
            "5135776b7a0640c9afd028c7931f068b",
            "eef727486fe94cb6b51194b1d0717d5a",
            "c0f9230635d148f3b0020722aededa0c",
            "afb6567472fd474a86f27ed785d18453",
            "86d58a009f994101916ce36fa6f3c2be",
            "6ebef002e08f4ab6aef300805ecb09fe",
            "9b60c790e1fb4ca7afb80f7e6b85df1a",
            "73e6c3c9dc76461c83fcae3b72d2e7d0",
            "8a0237c505b04a38bb8573b92651f2c9",
            "1ae09c2304264ba18e56fdd12b656a30",
            "bbaa4c6d8ad8438c95575a25d102a936",
            "cba48eb6faca4b99a3523c85cb6feeed",
            "398f893824cf46189526e60d8256c0d9",
            "995ba8b5054d4074957a4ad6961d4c2f",
            "87784e6e5a3f407f86708205d78e63cd",
            "9ba4797138be46eabbfcf387fa442b8f",
            "f70c9fea9169431686f6e0c10f6305f2",
            "78988bbcd40a4e279487c4b16a17ba12",
            "9b97123b79034db4bcabc1ae8b0ec4d8",
            "e425e16687374d91ae466640ef565ee0",
            "c331e9d751384f918967178c3d5ff359",
            "fe7d4980f3d04ecaa84f480b9cf848ff",
            "7892ee95d46f43958c7edffafb8f91f6",
            "c6ff0e0bacad41c186faa84232c36292",
            "7cca2316e2054ee5a17b9ed34c1b0663",
            "1a6ff00d515c4934b36bc0bbc795a449",
            "7e71022590c04a7c9f1b1fbb298b88e6",
            "36f3a13fb1da439ba09bf4cb8eebfbe9",
            "63b9db3b37e342278bd4c506170e0b81",
            "bd9ffdd5226a4a15a546f8123c86f39d",
            "1906049cca034cedbfe34d56159b7af3",
            "60e522fe0da84cbbb56f56666c37249b",
            "91d4796259bd43a8b1ab73d21ae051f1",
            "7244cc74f9b94ee1b8b07f8fac51efd9",
            "978412ffa319434084032ac45b1cd618",
            "72e37476f28143deb76ca56cf514d379",
            "a905d9b3364543b1b2eb155cea5f4105",
            "f6a64f0a5fd045e098d3f011c8346244",
            "f87c5e4134e648d5812ea15e62029351",
            "375bdfc699154cabab93d4c4317227b5",
            "2039d004198d4d759f68e13eccc5aef0",
            "243a534249aa4a22a93c5ef70c289db7",
            "1fbee335b823479192406c1b911c4cfb"
          ]
        },
        "outputId": "7a086183-65f4-4880-e851-88e3a5f99251"
      },
      "source": [
        "# XLNetTokenizer (for English)\n",
        "# tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=True)\n",
        "\n",
        "# AutoTokenizer (for Chinese)\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"hfl/chinese-xlnet-base\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/19.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "12ec24c4f7fd4ac289a9449d6635d822"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/671 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "383d55ed3cce4256a6c0e7737dc7b630"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/675k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "754575e2661f4c46ba278bfc1b495898"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.21M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bbaa4c6d8ad8438c95575a25d102a936"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/2.00 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fe7d4980f3d04ecaa84f480b9cf848ff"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/202 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "91d4796259bd43a8b1ab73d21ae051f1"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D3Xq-qP54SiF"
      },
      "source": [
        "# Texts for classification\n",
        "train_text_list = train['name'].values\n",
        "test_text_list = test['name'].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a0WBOEzcM0iR",
        "outputId": "bd943779-30f6-47b0-8610-3973b9312237",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "# train\n",
        "plot_sentence_embeddings_length(train_text_list, tokenizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAE9CAYAAAD9MZD2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeCElEQVR4nO3de7glVXnn8e+PmwpCWqQlSNNpQNCHJEZJSzA4BiFRBGM7hhgMGjSMnRiiOJpRJFHMOESMo0biLT1KbEcCQVQgQgxIuJhEgeYid0ILKM0A3cYL4AVE3vmj6pht03269u6zz6XO9/M8+zlVq6p2vbtOn373WrVqrVQVkiRpbttipgOQJEmbz4QuSVIPmNAlSeoBE7okST1gQpckqQdM6JIk9cBWMx3A5thpp51qyZIlMx2GJEnT5sorr/xmVS1cv3xOJ/QlS5awatWqmQ5DkqRpk+TrGyq3yV2SpB4woUuS1AMmdEmSesCELklSD5jQJUnqARO6JEk9YEKXJKkHTOiSJPWACV2SpB4woUuS1ANjS+hJTkmyNsn165W/LsnNSW5I8pcD5W9NsjrJLUleMK64JEnqo3GO5f4J4IPAJycKkjwPWAb8UlU9mORJbfk+wBHAzwNPBr6YZO+q+vEY45uTlhx3bqf97jjpsDFHIkmaTcZWQ6+qS4FvrVf8WuCkqnqw3WdtW74MOL2qHqyq24HVwH7jik2SpL6Z7nvoewP/JcllSS5J8qy2fFfgzoH91rRlkiSpg+mePnUrYEdgf+BZwBlJ9hjmDZIsB5YDLF68eMoDlCRpLpruGvoa4LPVuBx4BNgJuAvYbWC/RW3Zo1TViqpaWlVLFy581PzukiTNS9Od0M8CngeQZG9gG+CbwDnAEUkek2R3YC/g8mmOTZKkOWtsTe5JTgMOBHZKsgY4ATgFOKV9lO0h4KiqKuCGJGcANwIPA8fYw12SpO7GltCr6uUb2fSKjex/InDiuOKRJKnPHClOkqQeMKFLktQDJnRJknrAhC5JUg+Y0CVJ6gETuiRJPWBClySpB0zokiT1gAldkqQeMKFLktQDJnRJknrAhC5JUg+Y0CVJ6gETuiRJPWBClySpB0zokiT1gAldkqQeMKFLktQDJnRJknrAhC5JUg+Y0CVJ6gETuiRJPWBClySpB8aW0JOckmRtkus3sO1NSSrJTu16kpycZHWSa5PsO664JEnqo3HW0D8BHLJ+YZLdgOcD3xgofiGwV/taDnxkjHFJktQ7Y0voVXUp8K0NbHo/8GagBsqWAZ+sxleABUl2GVdskiT1zbTeQ0+yDLirqr663qZdgTsH1te0ZZIkqYOtputESbYFjqdpbt+c91lO0yzP4sWLpyAySZLmvumsoe8J7A58NckdwCLgqiQ/C9wF7Daw76K27FGqakVVLa2qpQsXLhxzyJIkzQ3TltCr6rqqelJVLamqJTTN6vtW1T3AOcDvtb3d9we+W1V3T1dskiTNdeN8bO004MvAU5OsSXL0JLufB9wGrAb+D/BH44pLkqQ+Gts99Kp6+Sa2LxlYLuCYccUiSVLfOVKcJEk9YEKXJKkHTOiSJPWACV2SpB4woUuS1AMmdEmSesCELklSD5jQJUnqARO6JEk9MG2zrWn+WHLcuZ32u+Okw8YciSTNH9bQJUnqARO6JEk9YEKXJKkHTOiSJPWACV2SpB4woUuS1AMmdEmSesCELklSD5jQJUnqARO6JEk9YEKXJKkHTOiSJPWACV2SpB4YW0JPckqStUmuHyh7T5Kbk1yb5HNJFgxse2uS1UluSfKCccUlSVIfjbOG/gngkPXKLgB+oaqeDvw78FaAJPsARwA/3x7z4SRbjjE2SZJ6ZWwJvaouBb61Xtn5VfVwu/oVYFG7vAw4vaoerKrbgdXAfuOKTZKkvpnJe+i/D/xju7wrcOfAtjVtmSRJ6mBGEnqSPwUeBk4d4djlSVYlWbVu3bqpD06SpDlo2hN6klcBLwKOrKpqi+8CdhvYbVFb9ihVtaKqllbV0oULF441VkmS5oqhEnqSLZLsMOrJkhwCvBl4cVV9f2DTOcARSR6TZHdgL+DyUc8jSdJ8s8mEnuTvkuyQZDvgeuDGJP+jw3GnAV8GnppkTZKjgQ8C2wMXJLkmyUcBquoG4AzgRuALwDFV9eORP5UkSfPMVh322aeq7ktyJE0ntuOAK4H3THZQVb18A8Ufn2T/E4ETO8QjSZLW06XJfeskWwMvAc6pqh+NOSZJkjSkLgn9b4A7gO2AS5P8HPDdcQYlSZKG0yWh/0NV7VpVh7a90r9B8wy5JEmaJbok9M8MrrRJ/fTxhCNJkkax0U5xSZ5GM7b6zyR56cCmHYDHjjswSZLU3WS93J9KMwDMAuA3B8rvB14zzqAkSdJwNprQq+ps4Owkz66qL09jTJIkaUhdnkNfneR4YMng/lVlxzhJkmaJLgn9bOBLwBcBR2+TJGkW6pLQt62qt4w9EkmSNLIuj619PsmhY49EkiSNrEtCP5Ymqf8wyX1J7k9y37gDkyRJ3W2yyb2qtp+OQCRJ0ui6TJ+aJK9I8rZ2fbck+40/NEmS1FWXJvcPA88GfrddfwD40NgikiRJQ+vSy/1XqmrfJFcDVNW3k2wz5rgkSdIQutTQf5RkS6AAkiwEHhlrVJIkaShdEvrJwOeAJyU5EfgX4C/GGpUkSRpKl17upya5EjgYCPCSqrpp7JFJkqTOutxDB7iXZvjXrYDHJdm3qq4aX1iSJGkYm0zoSd4JvAr4Gu199PbnQeMLS5IkDaNLDf1lwJ5V9dC4g5EkSaPp0inuemDBuAORJEmj65LQ3wVcneSfkpwz8drUQUlOSbI2yfUDZTsmuSDJre3PJ7TlSXJyktVJrk2y7+gfSZKk+adLk/tK4N3AdQz3/PkngA8CnxwoOw64sKpOSnJcu/4W4IXAXu3rV4CPtD8lSVIHXRL696vq5GHfuKouTbJkveJlwIHt8krgYpqEvgz4ZFUV8JUkC5LsUlV3D3teSZLmoy4J/UtJ3gWcAzw4UTjiY2s7DyTpe4Cd2+VdgTsH9lvTlpnQJUnqoEtCf2b7c/+Bss1+bK2qKkltes+flmQ5sBxg8eLFmxOCJEm90WWkuOdN4fnunWhKT7ILsLYtvwvYbWC/RW3ZhuJZAawAWLp06dBfCCRJ6qMuA8ssAH4PWDK4f1W9foTznQMcBZzU/jx7oPyPk5xO0xnuu94/lySpuy5N7ucBX2HIXu5JTqPpALdTkjXACTSJ/IwkRwNfpxm0ZuIchwKrge8Dr+56HkmS1C2hP7aq3jjsG1fVyzey6eAN7FvAMcOeQ5IkNboMLPN/k7wmyS7twDA7Jtlx7JFJkqTOutTQHwLeA/wpPz05yx7jCkqSJA2nS0J/E/CUqvrmuIORJEmj6dLkPtFRTZIkzVJdaujfA65JchE/PVLcKI+tSZKkMeiS0M9qX5IkaZbqMlLcyiTbAHu3RbdU1Y/GG5YkSRpGl5HiDqSZGe0OIMBuSY6qqkvHG5okSeqqS5P7e4HnV9UtAEn2Bk4DfnmcgUmSpO669HLfeiKZA1TVvwNbjy8kSZI0rC419FVJPgZ8ql1/BbBqfCFJkqRhdUnor6UZZ33iMbVLgY+MLSJJkjS0jSb0JAuBhVV1I/C+9kWSnwd2ANZNS4SSJGmTJruH/tfAThso3xH4wHjCkSRJo5gsoT9lQ4+mVdWXgKePLyRJkjSsyRL69pNss5e7JEmzyGQJfXWSQ9cvTPJC4LbxhSRJkoY1WS/3NwDnJnkZcGVbthR4NvCicQcmSZK622gNvapuBX4RuARY0r4uAZ7eDi4jSZJmiUmfQ6+qB4G/naZYJEnSiLoM/SpJkmY5E7okST2w0YSe5ML257unLxxJkjSKye6h75LkV4EXJzmdZi70n6iqq0Y9aZL/Dvw3oIDrgFcDuwCnA0+k6VX/yqp6aNRzSJI0n0yW0N8OvA1YRDuO+4ACDhrlhEl2pZnoZZ+q+kGSM4AjgEOB91fV6Uk+ChyNk8BIktTJRhN6VZ0JnJnkbVX1zjGc93FJfgRsC9xN8wXhd9vtK4F3YEKXJKmTTU6fWlXvTPJi4Llt0cVV9flRT1hVdyX538A3gB8A59M0sX+nqh5ud1sD7DrqOSRJmm822cs9ybuAY4Eb29exSf5i1BMmeQKwDNgdeDKwHXDIEMcvT7Iqyap165zBVZIk6FBDBw4DnlFVjwAkWQlcDRw/4jl/Hbi9qta17/dZ4ABgQZKt2lr6IuCuDR1cVSuAFQBLly6tEWOQJKlXuj6HvmBg+Wc285zfAPZPsm2SAAfT1PwvAg5v9zkKOHszzyNJ0rzRpYb+LuDqJBfRPLr2XOC4UU9YVZclORO4CniYpra/AjgXOD3J/2rLPj7qOSRJmm+6dIo7LcnFwLPaordU1T2bc9KqOgE4Yb3i24D9Nud9JUmar7rU0Kmqu4FzxhyLJEkakWO5S5LUAyZ0SZJ6YNKEnmTLJDdPVzCSJGk0kyb0qvoxcEuSxdMUjyRJGkGXTnFPAG5IcjnwvYnCqnrx2KLSZlty3LlT/p53nHTYlL+nJGlqdEnobxt7FJIkabN0eQ79kiQ/B+xVVV9Msi2w5fhDkyRJXXWZnOU1wJnA37RFuwJnjTMoSZI0nC6PrR1DM3nKfQBVdSvwpHEGJUmShtMloT9YVQ9NrCTZCnCWM0mSZpEuCf2SJMcDj0vyG8CngX8Yb1iSJGkYXRL6ccA64DrgD4DzgD8bZ1CSJGk4XXq5P5JkJXAZTVP7LVVlk7skSbPIJhN6ksOAjwJfo5kPffckf1BV/zju4CRJUjddBpZ5L/C8qloNkGRP4FzAhC5J0izR5R76/RPJvHUbcP+Y4pEkSSPYaA09yUvbxVVJzgPOoLmH/tvAFdMQmyRJ6miyJvffHFi+F/i1dnkd8LixRSRJkoa20YReVa+ezkA0/3SdEc5Z3iRp07r0ct8deB2wZHB/p0+VJGn26NLL/Szg4zSjwz0y3nAkSdIouiT0H1bVyWOPRJIkjaxLQv9AkhOA84EHJwqr6qpRT5pkAfAx4Bdoes7/PnAL8Pc0Tft3AC+rqm+Peg5JkuaTLgn9F4FXAgfxn03u1a6P6gPAF6rq8CTbANsCxwMXVtVJSY6jGUP+LZtxDkmS5o0uCf23gT0Gp1DdHEl+Bngu8CqA9n0fSrIMOLDdbSVwMSZ0SZI66TJS3PXAgik85+40z7L/bZKrk3wsyXbAzlV1d7vPPcDOU3hOSZJ6rUsNfQFwc5Ir+Ol76KM+trYVsC/wuqq6LMkHaJrXf6KqKskGZ3RLshxYDrB48eIRQ5AkqV+6JPQTpvica4A1VXVZu34mTUK/N8kuVXV3kl2AtRs6uKpWACsAli5d6jSukiTRbT70S6byhFV1T5I7kzy1qm4BDgZubF9HASe1P8+eyvPOdl1HTZMkaUO6jBR3P02vdoBtgK2B71XVDptx3tcBp7Y93G8DXk1zP/+MJEcDXwdethnvL0nSvNKlhr79xHKSAMuA/TfnpFV1DbB0A5sO3pz3lSRpvurSy/0nqnEW8IIxxSNJkkbQpcn9pQOrW9DUrH84togkSdLQuvRyH5wX/WGaYVmXjSUaSZI0ki730J0XXZKkWW6jCT3J2yc5rqrqnWOIR5IkjWCyGvr3NlC2HXA08ETAhC5J0iyx0YReVe+dWE6yPXAszfPipwPv3dhxkiRp+k16Dz3JjsAbgSNpZkDb1znKJUmafSa7h/4e4KU046b/YlU9MG1RaVaab8PTDvN57zjpsDFGIkmbNtnAMm8Cngz8GfD/ktzXvu5Pct/0hCdJkrqY7B76UKPISZKkmWPSliSpB0zokiT1gAldkqQeMKFLktQDJnRJknrAhC5JUg+Y0CVJ6gETuiRJPbDJ+dC1eebbcKmSpJlhDV2SpB4woUuS1AM2uWvW63rbwhnPJM1nM1ZDT7JlkquTfL5d3z3JZUlWJ/n7JNvMVGySJM01M9nkfixw08D6u4H3V9VTgG8DR89IVJIkzUEzktCTLAIOAz7Wrgc4CDiz3WUl8JKZiE2SpLlopmrofwW8GXikXX8i8J2qerhdXwPsOhOBSZI0F017Qk/yImBtVV054vHLk6xKsmrdunVTHJ0kSXPTTNTQDwBenOQO4HSapvYPAAuSTPS6XwTctaGDq2pFVS2tqqULFy6cjnglSZr1pj2hV9Vbq2pRVS0BjgD+uaqOBC4CDm93Owo4e7pjkyRprppNA8u8BXhjktU099Q/PsPxSJI0Z8zowDJVdTFwcbt8G7DfTMYjSdJcNZtq6JIkaUQmdEmSesCELklSDzg5i3rDSVwkzWfW0CVJ6gETuiRJPWBClySpB0zokiT1gAldkqQeMKFLktQDJnRJknrAhC5JUg84sIzmna4D0EjSXGINXZKkHjChS5LUAyZ0SZJ6wIQuSVIP2CluRHas0iBnepM006yhS5LUA9bQpTnMlgFJE6yhS5LUAyZ0SZJ6wIQuSVIPTHtCT7JbkouS3JjkhiTHtuU7Jrkgya3tzydMd2ySJM1VM1FDfxh4U1XtA+wPHJNkH+A44MKq2gu4sF2XJEkdTHtCr6q7q+qqdvl+4CZgV2AZsLLdbSXwkumOTZKkuWpG76EnWQI8E7gM2Lmq7m433QPsPENhSZI058xYQk/yeOAzwBuq6r7BbVVVQG3kuOVJViVZtW7dummIVJKk2W9GEnqSrWmS+alV9dm2+N4ku7TbdwHWbujYqlpRVUuraunChQunJ2BJkma5mejlHuDjwE1V9b6BTecAR7XLRwFnT3dskiTNVTMx9OsBwCuB65Jc05YdD5wEnJHkaODrwMtmIDZJkuakaU/oVfUvQDay+eDpjEWSpL5wchZpGjmZiqRxcehXSZJ6wIQuSVIPmNAlSeoBE7okST1gQpckqQdM6JIk9YAJXZKkHvA59AFdnxGWJGm2sYYuSVIPmNAlSeoBE7okST3gPXRpFpqp/hyONS/NXdbQJUnqAWvokmYFWwekzWMNXZKkHrCGLs0DjrEg9Z8JXdKcMtVfTqa6CX+Y+Lx9oKlkk7skST1gDV3S0OZjLdTbFprtrKFLktQDJnRJknrAJndJY2VT9ebzGX11Metq6EkOSXJLktVJjpvpeCRJmgtmVQ09yZbAh4DfANYAVyQ5p6punNnIJGnq2XqhqTSrEjqwH7C6qm4DSHI6sAwwoUsaiz4l1dn+jH6fzMbbILOtyX1X4M6B9TVtmSRJmsRsq6FvUpLlwPJ29YEkt8xkPHPUTsA3ZzqInvBaTh2v5dSZkmuZd09BJHPfZl3LMV3Dn9tQ4WxL6HcBuw2sL2rLfqKqVgArpjOovkmyqqqWznQcfeC1nDpey6njtZw6c+lazrYm9yuAvZLsnmQb4AjgnBmOSZKkWW9W1dCr6uEkfwz8E7AlcEpV3TDDYUmSNOvNqoQOUFXnAefNdBw95y2LqeO1nDpey6njtZw6c+ZapqpmOgZJkrSZZts9dEmSNAITes8lOSXJ2iTXD5TtmOSCJLe2P58wkzHOFUl2S3JRkhuT3JDk2Lbc6zmkJI9NcnmSr7bX8s/b8t2TXNYO/fz3bedYbUKSLZNcneTz7brXcQRJ7khyXZJrkqxqy+bM37cJvf8+ARyyXtlxwIVVtRdwYbuuTXsYeFNV7QPsDxyTZB+8nqN4EDioqn4JeAZwSJL9gXcD76+qpwDfBo6ewRjnkmOBmwbWvY6je15VPWPgUbU58/dtQu+5qroU+NZ6xcuAle3ySuAl0xrUHFVVd1fVVe3y/TT/ge6K13No1XigXd26fRVwEHBmW+617CDJIuAw4GPtevA6TqU58/dtQp+fdq6qu9vle4CdZzKYuSjJEuCZwGV4PUfSNhNfA6wFLgC+Bnynqh5ud3Ho527+Cngz8Ei7/kS8jqMq4PwkV7ajksIc+vuedY+taXpVVSXxUYchJHk88BngDVV1X1Mhang9u6uqHwPPSLIA+BzwtBkOac5J8iJgbVVdmeTAmY6nB55TVXcleRJwQZKbBzfO9r9va+jz071JdgFof66d4XjmjCRb0yTzU6vqs22x13MzVNV3gIuAZwMLkkxUNB419LMe5QDgxUnuAE6naWr/AF7HkVTVXe3PtTRfMvdjDv19m9Dnp3OAo9rlo4CzZzCWOaO9N/lx4Kaqet/AJq/nkJIsbGvmJHkc8Bs0fRIuAg5vd/NabkJVvbWqFlXVEpqhsv+5qo7E6zi0JNsl2X5iGXg+cD1z6O/bgWV6LslpwIE0MwbdC5wAnAWcASwGvg68rKrW7zin9SR5DvAl4Dr+837l8TT30b2eQ0jydJoORlvSVCzOqKr/mWQPmprmjsDVwCuq6sGZi3TuaJvc/6SqXuR1HF57zT7Xrm4F/F1VnZjkicyRv28TuiRJPWCTuyRJPWBClySpB0zokiT1gAldkqQeMKFLktQDJnTNO0ke2PRem/X+b0iy7VScL8ljknyxnf3pdzaw/U+S3NxuvyLJ7416rnFLsiDJH02y/cft55h4dZ4EI8mBEzONjRjbRo9vZ+DaqV3+t1HPIY2bQ79KU+8NwKeA70/Bez0ToKqesf6GJH9IMyDLfu0QtDsA/3UKzjkuC4A/Aj68ke0/2NDnnE2q6ldnOgZpY6yhS0CSPZN8oZ2U4UtJntaWfyLJyUn+LcltSQ5vy7dI8uG2dnxBkvOSHJ7k9cCTgYuSXDTw/ie2c39/JcmjJndo51w+K8m17T5Pb8eT/hTwrLbGuud6hx0PvLaq7gOoqvuqamX7fge382Nfl+SUJI9py+9I8q6J+Z6T7Jvkn5J8rf2CMFFbvSTJ2e1nPinJkWnmL79uIo52tLfPtC0DVyQ5oC1/R3vOi9vjX9/GexKwZ3vu9wzxu9lkzK0dkpyb5JYkH02yRXv885N8OclVST6dZix+khzS/v6uAl46cL4nJjk/zTztHwMysO2BgWt0cZIz2/c4NWkG9U9yaFt2ZftvZ2KO8l8baH24Ou2oZNKUqSpfvubVC3hgA2UXAnu1y79CM4QmNPPJf5rmy+8+wOq2/HDgvLb8Z2nmnD683XYHsNPAexfwm+3yXwJ/toHz/zVwQrt8EHBNu3wg8PkN7L8D8O2NfL7HAncCe7frn6SZSGYitte2y+8HrgW2BxYC9w6c8zvALsBjaMYB//N227HAX7XLf0czmQU0o2jd1C6/A/i39tidgP+gmR51CXD9JL+XHwPXDLx+Z8iYfwjsQTP63AXt72gn4FJgu3a/twBvH7hGe9Ek7DMmrjNwMvD2dvmw9ve30+C/nfZ836UZJ30L4MvAcwbed/d2v9MG3vcfgAPa5ccDW83034Kvfr1scte819bYfhX4dP5z5rTHDOxyVlU9Atw4ULt+DvDptvyewdr4BjwETNyfvZKmmXx9zwF+C6Cq/rmtJe4w0geCpwK3V9W/t+srgWNoptmEZmxqaIawfXw1c7vfn+TBtOOrA1dUO2Vkkq8B5w8c87x2+deBfQau2Q4TtV/g3GqGGn0wyVq6TTk5WZN7l5gvr6rb2phPo7mmP6T5IvavbZzb0CTfp9Fco1vb/T8FTEyX+VzaGntVnZvk2xuJ6fKqWtMefw3NF5YHgNuq6vZ2n9MG3vdfgfclORX47MSx0lQxoUtNDes7kySTwTGws5F9JvOjqpoYY/nHTMHfXTX3zB9IssdEEhvCxOd5hJ/+bI8MxLZ++YMb2GcLYP+q+uHgm7eJc/D4qfjMXWJefxzrovl9XVBVL18vxqm4Vz/UZ6yqk5KcCxxK8wXjBVV182THSMPwHrrmvWruQd+e5LehmVUtyS9t4rB/BX4rzb30nWmaYCfcT9MkPIwvAUe25z8Q+GYb12TeBXxooiaf5PFpernfAixJ8pR2v1cClwwZTxfnA6+bWOmQJEe5LsPYL8nu7b3z3wH+BfgKcMDEtUgzo9bewM0012iiX8Jgwr8U+N12/xcCTxgihluAPZIsadd/8mRCkj2r6rqqejdwBc7/rilmQtd8tG2SNQOvN9Ik06OTfBW4AVi2iff4DLAGuJGm49pVNPdUAVYAX9hEM/z63gH8cpJraTqPHTX57gB8hGaazCuSXE/zpeCRtsb8appbCBMzw310iFi6ej2wNE1HvhuBP5xs56r6D5qa6fUb6RT3uPz0Y2snDRnPFcAHaaZhvR34XFWtA14FnNZe2y8DT2uv0XLg3LZT3OAc138OPDfJDTRN79/oGkBV/YCmJ/8XklxJ8yVm4t/FG9rPfi3wI+Afh/x80qScbU0aUZLHV9UDaaZXvJymw9M9Mx2XZtbAv4sAHwJurar3z3Rc6j/voUuj+3zbIWsb4J0mc7Vek+Qomn8XVwN/M8PxaJ6whi5JUg94D12SpB4woUuS1AMmdEmSesCELklSD5jQJUnqARO6JEk98P8BWkewYmqcRfIAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mYR3yDcpM2V7",
        "outputId": "78f2a5eb-e65e-49a8-a0ef-c9de95246215",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "# test\n",
        "plot_sentence_embeddings_length(test_text_list, tokenizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAE9CAYAAAD9MZD2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcjklEQVR4nO3deZgldX3v8feHAVwQRJ0xGmAcVNSLN95IJrhexR3QQK7igkZxuRKNRHxMvCEuSORJFH3USC4uRL0SNRCXiKOMghoEo6IzLLIjA5IIUUElLBpA5Hv/qGo9tH266zRT3T3V79fznGeq6vxOnW+dZT5dy/n9UlVIkqQt21aLXYAkSbrjDHRJkgbAQJckaQAMdEmSBsBAlyRpAAx0SZIGYOvFLmBSK1eurDVr1ix2GZIkLYgzzzzzx1W1aq52W1ygr1mzho0bNy52GZIkLYgk/9alnYfcJUkaAANdkqQBMNAlSRoAA12SpAEw0CVJGgADXZKkATDQJUkagN4CPcmHk1yd5Pwx9yfJ0Uk2JTk3yR591SJJ0tD1uYf+EWDvWe7fB9itvR0MvK/HWiRJGrTeAr2qTgd+OkuT/YF/qMYZwI5J7ttXPZIkDdlinkPfCfj+yPyV7TJJkjShLaIv9yQH0xyWZ/Xq1Zt13WsOO6lz2yve9vTNus6u61ssfbw2m9tivdZ+bsbzc6P58HNzxy3mHvpVwC4j8zu3y35DVR1bVWurau2qVXMOOCNJ0rKzmIG+DnhRe7X7I4HrquoHi1iPJElbrN4OuSc5HtgLWJnkSuDNwDYAVfV+YD2wL7AJ+Dnwkr5qkSRp6HoL9Ko6cI77C3hVX88vSdJyYk9xkiQNgIEuSdIAGOiSJA2AgS5J0gAY6JIkDYCBLknSABjokiQNgIEuSdIAGOiSJA2AgS5J0gAY6JIkDYCBLknSABjokiQNgIEuSdIAGOiSJA2AgS5J0gAY6JIkDYCBLknSABjokiQNgIEuSdIAGOiSJA2AgS5J0gAY6JIkDYCBLknSABjokiQNgIEuSdIAGOiSJA2AgS5J0gAY6JIkDYCBLknSABjokiQNgIEuSdIAGOiSJA2AgS5J0gAY6JIkDYCBLknSABjokiQNgIEuSdIAGOiSJA2AgS5J0gAY6JIkDYCBLknSABjokiQNQK+BnmTvJJck2ZTksBnuX53k1CRnJzk3yb591iNJ0lD1FuhJVgDHAPsAuwMHJtl9WrM3Ap+oqocDzwPe21c9kiQNWZ976HsCm6rq8qq6BTgB2H9amwJ2aKfvDvxHj/VIkjRYW/e47p2A74/MXwk8YlqbI4BTkvwpsB3w5B7rkSRpsBb7orgDgY9U1c7AvsBHk/xGTUkOTrIxycZrrrlmwYuUJGmp6zPQrwJ2GZnfuV026mXAJwCq6pvAnYGV01dUVcdW1dqqWrtq1aqeypUkacvVZ6BvAHZLsmuSbWkuels3rc2/A08CSPLfaALdXXBJkibUW6BX1a3AIcDJwEU0V7NfkOQtSfZrm/0Z8PIk3wGOB15cVdVXTZIkDVWfF8VRVeuB9dOWHT4yfSHwmD5rkCRpOVjsi+IkSdJmYKBLkjQABrokSQNgoEuSNAAGuiRJA2CgS5I0AAa6JEkDYKBLkjQABrokSQNgoEuSNAAGuiRJA2CgS5I0ABMFepKtkuzQVzGSJGl+5gz0JP+YZIck2wHnAxcmeV3/pUmSpK667KHvXlXXA38IfAHYFXhhr1VJkqSJdAn0bZJsQxPo66rqFz3XJEmSJtQl0D8AXAFsB5ye5H7AdX0WJUmSJtMl0D9XVTtV1b5VVcC/Ay/tuS5JkjSBLoH+6dGZNtRP6KccSZI0H1uPuyPJQ4CHAndP8syRu3YA7tx3YZIkqbuxgQ48GHgGsCPwByPLbwBe3mdRkiRpMmMDvao+C3w2yaOq6psLWJMkSZrQbHvoUzYleT2wZrR9VXlhnCRJS0SXQP8s8DXgy8Av+y1HkiTNR5dAv2tV/UXvlUiSpHnr8rO1zyfZt/dKJEnSvHUJ9ENpQv2mJNcnuSHJ9X0XJkmSupvzkHtVbb8QhUiSpPnrMnxqkvxRkje187sk2bP/0iRJUlddDrm/F3gU8Px2/kbgmN4qkiRJE+tylfsjqmqPJGcDVNW1SbbtuS5JkjSBLnvov0iyAiiAJKuA23qtSpIkTaRLoB8NfAa4d5K/Bv4V+Jteq5IkSRPpcpX7x5OcCTwJCPCHVXVR75VJkqTOupxDB/gRTfevWwN3SbJHVZ3VX1mSJGkScwZ6kiOBFwOX0Z5Hb/99Yn9lSZKkSXTZQ38O8ICquqXvYiRJ0vx0uSjufGDHvguRJEnz12UP/a3A2UnOB26eWlhV+/VWlSRJmkiXQD8OOAo4D39/LknSktQl0H9eVUf3XokkSZq3LoH+tSRvBdZx+0Pu/mxNkqQlokugP7z995Ejy/zZmiRJS0iXnuKeMN+VJ9kbeA+wAvhgVb1thjbPAY6g+SPhO1X1/OltJEnS7Lp0LLMj8CJgzWj7qnr1HI9bQTPM6lOAK4ENSdZV1YUjbXYD/hJ4TDuK273nsxGSJC13XQ65rwfOYPKr3PcENlXV5QBJTgD2By4cafNy4Jiquhagqq6eYP2SJKnVJdDvXFWvnce6dwK+PzJ/JfCIaW0eBJDk6zSH5Y+oqi9OX1GSg4GDAVavXj2PUiRJGrYuPcV9NMnLk9w3yT2nbpvp+bcGdgP2Ag4E/r49xH87VXVsVa2tqrWrVq3aTE8tSdJwdNlDvwV4B/AGbj84y/3neNxVwC4j8zu3y0ZdCXyrqn4BfC/Jd2kCfkOHuiRJUqvLHvqfAQ+sqjVVtWt7myvMoQnl3ZLsmmRb4Hk0v2UfdSLN3jlJVtIcgr+8c/WSJAnoFuibgJ9PuuKquhU4BDgZuAj4RFVdkOQtSab6gT8Z+EmSC4FTgddV1U8mfS5Jkpa7Lofcfwack+RUbt9T3Kw/W2vbrKe5Sn502eEj0wW8tr1JkqR56hLoJ7Y3SZK0RHXpKe649hz4g9pFl7QXsUmSpCWiS09xe9EMoXoFEGCXJAdV1en9liZJkrrqcsj9ncBTq+oSgCQPAo4Hfq/PwiRJUnddrnLfZirMAarqu8A2/ZUkSZIm1WUPfWOSDwIfa+f/CNjYX0mSJGlSXQL9lcCrgKmfqZ0OvK+3iiRJ0sTGBnqSVcCqdrjTd7U3kjwU2AG4ZkEqlCRJc5rtHPrfAStnWH5P4D39lCNJkuZjtkB/4Ew/TauqrwEP668kSZI0qdkCfftZ7vMqd0mSlpDZAn1Tkn2nL0yyD46IJknSkjLbVe6vAU5K8hzgzHbZWuBRwDP6LkySJHU3dg+9qi4Ffgc4DVjT3k4DHtZ2LiNJkpaIWX+HXlU3A/9vgWqRJEnz1KXrV0mStMQZ6JIkDcDYQE/ylfbfoxauHEmSNB+znUO/b5JHA/slOYFmLPRfqaqzeq1MkiR1NlugHw68CdiZth/3EQU8sa+iJEnSZMYGelV9CvhUkjdV1ZELWJMkSZrQnMOnVtWRSfYDHtcu+mpVfb7fsiRJ0iTmvMo9yVuBQ4EL29uhSf6m78IkSVJ3c+6hA08HfreqbgNIchxwNvD6PguTJEnddf0d+o4j03fvoxBJkjR/XfbQ3wqcneRUmp+uPQ44rNeqJEnSRLpcFHd8kq8Cv98u+ouq+mGvVUmSpIl02UOnqn4ArOu5FkmSNE/25S5J0gAY6JIkDcCsgZ5kRZKLF6oYSZI0P7MGelX9ErgkyeoFqkeSJM1Dl4vi7gFckOTbwM+mFlbVfr1VJUmSJtIl0N/UexWSJOkO6fI79NOS3A/Yraq+nOSuwIr+S5MkSV11GZzl5cCngA+0i3YCTuyzKEmSNJkuP1t7FfAY4HqAqroUuHefRUmSpMl0CfSbq+qWqZkkWwPVX0mSJGlSXQL9tCSvB+6S5CnAJ4HP9VuWJEmaRJdAPwy4BjgP+GNgPfDGPouSJEmT6XKV+21JjgO+RXOo/ZKq8pC7JElLyJyBnuTpwPuBy2jGQ981yR9X1Rf6Lk6SJHXT5ZD7O4EnVNVeVfV44AnAu7usPMneSS5JsinJYbO0e1aSSrK2W9mSJGlUl0C/oao2jcxfDtww14OSrACOAfYBdgcOTLL7DO22Bw6lOaQvSZLmYewh9yTPbCc3JlkPfILmHPqzgQ0d1r0nsKmqLm/XdwKwP3DhtHZHAkcBr5usdEmSNGW2c+h/MDL9I+Dx7fQ1wF06rHsn4Psj81cCjxhtkGQPYJeqOimJgS5J0jyNDfSqekmfT5xkK+BdwIs7tD0YOBhg9WpHcpUkabouV7nvCvwpsGa0fYfhU68CdhmZ37ldNmV74L8DX00CcB9gXZL9qmrj6Iqq6ljgWIC1a9f6kzlJkqbpMnzqicCHaHqHu22CdW8Admv/ILgKeB7w/Kk7q+o6YOXUfJKvAn8+PcwlSdLcugT6TVV19KQrrqpbkxwCnEwz3OqHq+qCJG8BNlbVuknXKUmSZtYl0N+T5M3AKcDNUwur6qy5HlhV62m6ih1ddviYtnt1qEWSJM2gS6D/DvBC4In8+pB7tfOSJGkJ6BLozwbuPzqEqiRJWlq69BR3PrBj34VIkqT567KHviNwcZIN3P4c+lw/W5MkSQukS6C/ufcqJEnSHdJlPPTTFqIQSZI0f116iruB5qp2gG2BbYCfVdUOfRYmSZK667KHvv3UdJo+WvcHHtlnUZIkaTJdrnL/lWqcCDytp3okSdI8dDnk/syR2a2AtcBNvVUkSZIm1uUq99Fx0W8FrqA57C5JkpaILufQex0XXZIk3XFjAz3JjIOotKqqjuyhHkmSNA+z7aH/bIZl2wEvA+4FGOiSJC0RYwO9qt45NZ1ke+BQ4CXACcA7xz1OkiQtvFnPoSe5J/Ba4AXAccAeVXXtQhQmSZK6m+0c+juAZwLHAr9TVTcuWFWSJGkis3Us82fAbwNvBP4jyfXt7YYk1y9MeZIkqYvZzqFP1IucJElaPIa2JEkDYKBLkjQABrokSQNgoEuSNAAGuiRJA2CgS5I0AAa6JEkDYKBLkjQABrokSQNgoEuSNAAGuiRJA2CgS5I0AAa6JEkDYKBLkjQABrokSQNgoEuSNAAGuiRJA2CgS5I0AAa6JEkDYKBLkjQABrokSQNgoEuSNAAGuiRJA2CgS5I0AL0GepK9k1ySZFOSw2a4/7VJLkxybpKvJLlfn/VIkjRUvQV6khXAMcA+wO7AgUl2n9bsbGBtVT0M+BTw9r7qkSRpyPrcQ98T2FRVl1fVLcAJwP6jDarq1Kr6eTt7BrBzj/VIkjRYfQb6TsD3R+avbJeN8zLgCz3WI0nSYG292AUAJPkjYC3w+DH3HwwcDLB69eoFrEySpC1Dn3voVwG7jMzv3C67nSRPBt4A7FdVN8+0oqo6tqrWVtXaVatW9VKsJElbsj4DfQOwW5Jdk2wLPA9YN9ogycOBD9CE+dU91iJJ0qD1FuhVdStwCHAycBHwiaq6IMlbkuzXNnsHcDfgk0nOSbJuzOokSdIsej2HXlXrgfXTlh0+Mv3kPp9fkqTlwp7iJEkaAANdkqQBMNAlSRoAA12SpAEw0CVJGgADXZKkATDQJUkaAANdkqQBMNAlSRoAA12SpAEw0CVJGgADXZKkATDQJUkaAANdkqQBMNAlSRoAA12SpAEw0CVJGgADXZKkATDQJUkaAANdkqQBMNAlSRoAA12SpAEw0CVJGgADXZKkATDQJUkaAANdkqQBMNAlSRoAA12SpAEw0CVJGgADXZKkATDQJUkaAANdkqQBMNAlSRoAA12SpAEw0CVJGgADXZKkATDQJUkaAANdkqQBMNAlSRoAA12SpAEw0CVJGgADXZKkATDQJUkagF4DPcneSS5JsinJYTPcf6ck/9Te/60ka/qsR5Kkoeot0JOsAI4B9gF2Bw5Msvu0Zi8Drq2qBwLvBo7qqx5Jkoaszz30PYFNVXV5Vd0CnADsP63N/sBx7fSngCclSY81SZI0SH0G+k7A90fmr2yXzdimqm4FrgPu1WNNkiQNUqqqnxUnBwB7V9X/budfCDyiqg4ZaXN+2+bKdv6yts2Pp63rYODgdvbBwCWbsdSVwI/nbDUcbu9wLadtBbd3yJbTtsLc23u/qlo110q23nz1/IargF1G5ndul83U5sokWwN3B34yfUVVdSxwbB9FJtlYVWv7WPdS5PYO13LaVnB7h2w5bStsvu3t85D7BmC3JLsm2RZ4HrBuWpt1wEHt9AHAv1RfhwwkSRqw3vbQq+rWJIcAJwMrgA9X1QVJ3gJsrKp1wIeAjybZBPyUJvQlSdKE+jzkTlWtB9ZPW3b4yPRNwLP7rKGDXg7lL2Fu73Atp20Ft3fIltO2wmba3t4uipMkSQvHrl8lSRqAZRPoSa5Icl6Sc5JsnOH+JDm67Yb23CR7LEadm0OSB7fbOXW7PslrprXZK8l1I20OH7e+pSjJh5Nc3f70cWrZPZN8Kcml7b/3GPPYg9o2lyY5aKY2S8mYbX1Hkovbz+pnkuw45rGzfu6XojHbe0SSq0Y+r/uOeeys3U0vRWO2959GtvWKJOeMeewW9f4m2SXJqUkuTHJBkkPb5UP97o7b3n6+v1W1LG7AFcDKWe7fF/gCEOCRwLcWu+bNtN0rgB/S/I5xdPlewOcXu747sF2PA/YAzh9Z9nbgsHb6MOCoGR53T+Dy9t97tNP3WOztmce2PhXYup0+aqZtbe+b9XO/FG9jtvcI4M/neNwK4DLg/sC2wHeA3Rd7e+azvdPufydw+BDeX+C+wB7t9PbAd2m6Bh/qd3fc9vby/V02e+gd7A/8QzXOAHZMct/FLmozeBJwWVX922IXsjlV1ek0v4wYNdqV8HHAH87w0KcBX6qqn1bVtcCXgL17K3QzmGlbq+qUanpXBDiDpp+HQRjz3nbRpbvpJWe27W27wn4OcPyCFtWTqvpBVZ3VTt8AXETTY+hQv7szbm9f39/lFOgFnJLkzLbnuem6dFW7JXoe4/8zeFSS7yT5QpKHLmRRPfmtqvpBO/1D4LdmaDPE9/mlNEeXZjLX535Lckh7iPLDYw7JDvG9/Z/Aj6rq0jH3b7Hvb5rRNR8OfItl8N2dtr2jNtv3t9efrS0xj62qq5LcG/hSkovbv4wHq+3QZz/gL2e4+yyaw/A3tucjTwR2W8j6+lRVlWTwP+FI8gbgVuDjY5oM5XP/PuBImv/gjqQ5DP3SRa1oYRzI7HvnW+T7m+RuwKeB11TV9RkZk2uI393p2zuyfLN+f5fNHnpVXdX+ezXwGZrDc6O6dFW7pdkHOKuqfjT9jqq6vqpubKfXA9skWbnQBW5mP5o6TdL+e/UMbQbzPid5MfAM4AXVnnCbrsPnfotQVT+qql9W1W3A3zPzdgzmvQVI0x32M4F/GtdmS3x/k2xDE24fr6p/bhcP9rs7Znt7+f4ui0BPsl2S7aemaS5IOH9as3XAi9J4JHDdyCGgLdXYv+6T3Kc9P0eSPWk+C7/Rj/4WZrQr4YOAz87Q5mTgqUnu0R62fWq7bIuSZG/g/wD7VdXPx7Tp8rnfIky7nuV/MfN2dOluekvyZODiagevmm5LfH/b/3M+BFxUVe8auWuQ391x29vb93exrwJciBvNVa/faW8XAG9ol78CeEU7HeAYmqtkzwPWLnbdd3Cbt6MJ6LuPLBvd3kPa1+I7NBdlPHqxa55w+44HfgD8guZc2stoht79CnAp8GXgnm3btcAHRx77UmBTe3vJYm/LPLd1E835xHPa2/vbtr8NrG+nZ/zcL/XbmO39aPu9PJfmP//7Tt/edn5fmiuJL9uSt7dd/pGp7+tI2y36/QUeS3Pa5NyRz+6+A/7ujtveXr6/9hQnSdIALItD7pIkDZ2BLknSABjokiQNgIEuSdIAGOiSJA2Aga5lIcmNPa//NUnuujmeL8mdkny5HWHpuTPc/+ftSE3nJNmQ5EXzfa6+JdkxyZ/Mcv8vc/uRATuPkJZmxMDP34Haxj6+HeVqZTv9jfk+h7SQllPXr1KfXgN8DJixk4gJPRygqn53+h1JXgE8Bdizmi4zd6DpaGWp2hH4E+C9Y+7/r5m2cympqkcvdg1SF+6ha9lK8oAkX2wHPvhakoe0yz+S5Ogk30hyeZID2uVbJXlvu3f8pSTrkxyQ5NU0HUKcmuTUkfX/dTv4zRlJfmOwiTRjQJ/YDjhyRpKHtX02fwz4/XaP9QHTHvZ64JXV9gddTRe+x7Xre1KSs9OMn/zhJHdql1+R5K3t+jYm2SPJyUkua/9AmNpbPS3JZ9ttfluSFyT5dru+B7TtViX5dHtkYEOSx7TLj2if86vt41/d1vs24AHtc79jgvdmzppbOyQ5Kc0Y6O9PslX7+Kcm+WaSs5J8Mk1f2lPjpV+c5CyablWnnu9eSU5JM2b1B2k6mpq678aR1+irST7VruPjya96W9y3XXZm+9n5fLv88SNHH85O2/OX1IvF7knHm7eFuAE3zrDsK8Bu7fQjgH9ppz8CfJLmD97daYbkBDgAWN8uvw9wLXBAe98VjIxbTNM71B+0028H3jjD8/8d8OZ2+onAOe30XswwVj2wA3DtmO27M03PUw9q5/+BZiCIqdpe2U6/m6bXqu2BVTQjeU0953/SjN98J5o+sv+qve9Q4G/b6X+kGTACYDVNl5bQjFf+jfaxK2l6KdwGWMOYcb7bx/2SX/eWdQ7w3AlrvommR60VNMNpHtA+/+nAdm27vwAOH3mNdqMJ7E9Mvc7A0bRjjgNPb9+/laOfnfb5rqPpQ3wr4Js0PYFNrXfXtt3xI+v9HPCYdvputGNge/PWx81D7lqW2j22RwOfzK9HerrTSJMTqxkI5MKRvevHAp9sl/9wdG98BrcAU+dnz6Q5TD7dY4FnAVTVv7R7iTvMa4PgwcD3quq77fxxwKuAv23np/o0Pw+4WzVjM9+Q5OYkO7b3bah2/IIklwGnjDzmCe30k4HdR16zHab2foGTqupm4OYkVzPzEJjTzXbIvUvN366qy9uaj6d5TW+i+UPs622d29KE70NoXqNL2/YfA6aGpHwc7R57VZ2U5NoxNX272r7Vk5xD8wfLjcDlVfW9ts3xI+v9OvCuJB8H/rnG9MsubQ4GuparrYD/nCVMbh6Zzpg2s/lFVU31q/xLNsN3rZpz5jcmuf9UiE1gantu4/bbdttIbdOX3zxDm62AR1bVTaMrb4Nz9PGbY5u71Dy97+qieb++VFUHTqtxc5yrn2gbq+ptSU6i6b/760meVlUXb4Y6pN/gOXQtS9Wcg/5ekmdDMypSkv8xx8O+Djwrzbn036I5BDvlBppDwpP4GvCC9vn3An5cI2Mlj/FW4JipPfkkd0tzlfslwJokD2zbvRA4bcJ6ujgF+NOpmQ4hOZ/XZRJ7phldbSvgucC/0gw29Jip1yLNqFUPAi6meY2mrksYDfzTgee37fcB7jFBDZcA90+ypp3/1S8Tkjygqs6rqqNoRoN7yITbJ3VmoGu5uGuSK0dur6UJ05clmRrNaP851vFpmtGwLqS5cO0smnOqAMcCX5zjMPx0RwC/l+RcmovHDpq9OQDvA04FNiQ5n+aPgtvaPeaX0JxCOI9mL/b9E9TS1auBtWku5LuQZgS/sarqJzR7puePuSjuLrn9z9beNmE9G4D/C1wEfA/4TFVdA7wYOL59bb8JPKR9jQ4GTmovihsdc/uvgMcluYDm0Pu/dy2gqv6L5kr+LyY5k+aPmKnPxWvabT+XZjS1L0y4fVJnjrYmTSDJ3arqxiT3Ar5Nc8HTDxe7Li2ukc/F1DDMl1bVuxe7Li0vnkOXJvP59oKsbYEjDXO1Xp7kIJrPxdnABxa5Hi1D7qFLkjQAnkOXJGkADHRJkgbAQJckaQAMdEmSBsBAlyRpAAx0SZIG4P8D1eoQEpid7RcAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### (2) Tokenize input texts into ids\n",
        "*   Append the appropriate special characters to the end of the text to denote end of sentence.\n",
        "*   Truncate or pad the appropriate sequence length.\n",
        "\n"
      ],
      "metadata": {
        "id": "o0F5uInMMElZ"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ocveMTIfM5cP"
      },
      "source": [
        "# 這裡可能要再改\n",
        "def tokenize_inputs(text_list, tokenizer, num_embeddings=512):\n",
        "    # tokenize the text, then truncate sequence to the desired length minus 2 for the 2 special characters\n",
        "    tokenized_texts = list(map(lambda t: tokenizer.tokenize(t)[:num_embeddings-2], text_list))\n",
        "    # convert tokenized text into numeric ids for the appropriate LM\n",
        "    input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "    # append special token \"<s>\" and </s> to end of sentence\n",
        "    input_ids = [tokenizer.build_inputs_with_special_tokens(x) for x in input_ids]\n",
        "    # pad sequences\n",
        "    input_ids = pad_sequences(input_ids, maxlen=num_embeddings, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "    return input_ids"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p2ZVLj_OM8sV",
        "outputId": "cc5511b3-8ff1-4eef-948f-eb24d1d88f03",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Create input id tokens (train)\n",
        "train_input_ids = tokenize_inputs(train_text_list, tokenizer, num_embeddings=250)\n",
        "train_input_ids"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[   19,  2564, 15664, ...,     0,     0,     0],\n",
              "       [   19,  5473,    95, ...,     0,     0,     0],\n",
              "       [ 7801,   917, 17175, ...,     0,     0,     0],\n",
              "       ...,\n",
              "       [   19,  2400, 18718, ...,     0,     0,     0],\n",
              "       [   19,    42,    83, ...,     0,     0,     0],\n",
              "       [   19,  8114,  2114, ...,     0,     0,     0]])"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VrzAuQ_6M-ER",
        "outputId": "838b36ee-a7bf-44a5-d66e-c19450b2ba25",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Create input id tokens (test)\n",
        "test_input_ids = tokenize_inputs(test_text_list, tokenizer, num_embeddings=250)\n",
        "test_input_ids"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[   19,  1141, 11569, ...,     0,     0,     0],\n",
              "       [   19,    59, 23183, ...,     0,     0,     0],\n",
              "       [ 4531,  4615,    41, ...,     0,     0,     0],\n",
              "       ...,\n",
              "       [   19,  6064,   738, ...,     0,     0,     0],\n",
              "       [   19,  2140, 21679, ...,     0,     0,     0],\n",
              "       [   19,   971,   726, ...,     0,     0,     0]])"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### (3) Create attention masks\n",
        "*   Tell model whether attention should be applied to the input id tokens.\n",
        "*   Do not want to perform attention on padding tokens."
      ],
      "metadata": {
        "id": "lgpTNBiQNpsn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def create_attn_masks(input_ids):\n",
        "    # Create attention masks\n",
        "    attention_masks = []\n",
        "    # Create a mask of 1s for each token followed by 0s for padding\n",
        "    for seq in input_ids:\n",
        "        seq_mask = [float(i>0) for i in seq]\n",
        "        attention_masks.append(seq_mask)\n",
        "    return attention_masks"
      ],
      "metadata": {
        "id": "VBd7Su9INZuu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TgGLmaKKM_xB"
      },
      "source": [
        "# Create attention masks\n",
        "train_attention_masks = create_attn_masks(train_input_ids)\n",
        "test_attention_masks = create_attn_masks(test_input_ids)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### (4) Add input ids and attention masks to the dataframe"
      ],
      "metadata": {
        "id": "eIZIAwD6PDb8"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OQPuY4mONCSu"
      },
      "source": [
        "# train\n",
        "train[\"features\"] = train_input_ids.tolist()\n",
        "train[\"masks\"] = train_attention_masks\n",
        "# test\n",
        "test[\"features\"] = test_input_ids.tolist()\n",
        "test[\"masks\"] = test_attention_masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KjzFlRljNEkB",
        "outputId": "1e88acd9-ee26-4dd8-cc5c-9b5b28b484b8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 505
        }
      },
      "source": [
        "train.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          id                                       name  bb霜  cc霜  dd霜  rtd調酒  \\\n",
              "62423  62424           荷蘭blackrain男刷毛半門襟長袖上衣br-14176咖啡黑    0    0    0      0   \n",
              "43564  43565                                    娘家益生菌包盒    0    0    0      0   \n",
              "29397  29398     realbarrier沛麗膚屏護保濕潔顏乳洗完不緊繃修復肌膚屏障效期2021    0    0    0      0   \n",
              "51432  51433                                魅光極線抗暈眼線液筆-    0    0    0      0   \n",
              "59603  59604  nugens捷視科技pc-freehdmi800萬實物攝影機限量贈送記憶卡無線滑鼠    0    0    0      0   \n",
              "\n",
              "       中式香腸  乳酸菌  人工淚液  保久乳  ...  電鍋  電風扇  面膜  面霜/乳霜  香氛機  香氛蠟燭  鮮乳  麥片穀類  \\\n",
              "62423     0    0     0    0  ...   0    0   0      0    0     0   0     0   \n",
              "43564     0    0     0    0  ...   0    0   0      0    0     0   0     0   \n",
              "29397     0    0     0    0  ...   0    0   0      0    0     0   0     0   \n",
              "51432     0    0     0    0  ...   0    0   0      0    0     0   0     0   \n",
              "59603     0    0     0    0  ...   0    0   0      0    0     0   0     0   \n",
              "\n",
              "                                                features  \\\n",
              "62423  [19, 2564, 15664, 7542, 2901, 1430, 1012, 1060...   \n",
              "43564  [19, 5473, 95, 2118, 153, 2440, 970, 6759, 4, ...   \n",
              "29397  [7801, 917, 17175, 2883, 994, 8515, 1627, 2235...   \n",
              "51432  [19, 21020, 182, 1235, 191, 1952, 21566, 931, ...   \n",
              "59603  [19, 629, 10829, 8899, 2735, 1378, 1682, 1116,...   \n",
              "\n",
              "                                                   masks  \n",
              "62423  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "43564  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "29397  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "51432  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "59603  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "\n",
              "[5 rows x 220 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-efb9d7a1-ff4b-4c55-8f20-aebe37a76f6a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "      <th>bb霜</th>\n",
              "      <th>cc霜</th>\n",
              "      <th>dd霜</th>\n",
              "      <th>rtd調酒</th>\n",
              "      <th>中式香腸</th>\n",
              "      <th>乳酸菌</th>\n",
              "      <th>人工淚液</th>\n",
              "      <th>保久乳</th>\n",
              "      <th>...</th>\n",
              "      <th>電鍋</th>\n",
              "      <th>電風扇</th>\n",
              "      <th>面膜</th>\n",
              "      <th>面霜/乳霜</th>\n",
              "      <th>香氛機</th>\n",
              "      <th>香氛蠟燭</th>\n",
              "      <th>鮮乳</th>\n",
              "      <th>麥片穀類</th>\n",
              "      <th>features</th>\n",
              "      <th>masks</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>62423</th>\n",
              "      <td>62424</td>\n",
              "      <td>荷蘭blackrain男刷毛半門襟長袖上衣br-14176咖啡黑</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[19, 2564, 15664, 7542, 2901, 1430, 1012, 1060...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43564</th>\n",
              "      <td>43565</td>\n",
              "      <td>娘家益生菌包盒</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[19, 5473, 95, 2118, 153, 2440, 970, 6759, 4, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29397</th>\n",
              "      <td>29398</td>\n",
              "      <td>realbarrier沛麗膚屏護保濕潔顏乳洗完不緊繃修復肌膚屏障效期2021</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[7801, 917, 17175, 2883, 994, 8515, 1627, 2235...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51432</th>\n",
              "      <td>51433</td>\n",
              "      <td>魅光極線抗暈眼線液筆-</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[19, 21020, 182, 1235, 191, 1952, 21566, 931, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59603</th>\n",
              "      <td>59604</td>\n",
              "      <td>nugens捷視科技pc-freehdmi800萬實物攝影機限量贈送記憶卡無線滑鼠</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[19, 629, 10829, 8899, 2735, 1378, 1682, 1116,...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 220 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-efb9d7a1-ff4b-4c55-8f20-aebe37a76f6a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-efb9d7a1-ff4b-4c55-8f20-aebe37a76f6a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-efb9d7a1-ff4b-4c55-8f20-aebe37a76f6a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDcrK191NG_K",
        "outputId": "9c1ff673-2eab-4de8-adf2-f4ed5c100ec4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "test.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          id                               name  \\\n",
              "90077  90078              hairrecipe蜂蜜保濕營養護髮精華素   \n",
              "26263  26264                         一匙靈制菌洗衣精瓶裝   \n",
              "25412  25413  日本no1ariel日本進口三合一3d洗衣膠囊洗衣球室內晾乾六入箱   \n",
              "37753  37754              cnplaboratory蜂膠能量彈潤乳霜   \n",
              "89670  89671                       炫豔護髮染髮乳自然褐白染   \n",
              "\n",
              "                                                features  \\\n",
              "90077  [19, 1141, 11569, 1958, 5263, 5304, 5398, 6027...   \n",
              "26263  [19, 59, 23183, 1565, 299, 2440, 21816, 1191, ...   \n",
              "25412  [4531, 4615, 41, 8816, 1725, 161, 10046, 86, 1...   \n",
              "37753  [19, 4447, 1116, 2453, 18960, 14707, 592, 5398...   \n",
              "89670  [19, 17141, 21748, 2267, 4532, 6342, 4532, 325...   \n",
              "\n",
              "                                                   masks  \n",
              "90077  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "26263  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "25412  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "37753  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
              "89670  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-67f6faac-255e-47e2-83f5-dcfd99b1464a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "      <th>features</th>\n",
              "      <th>masks</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>90077</th>\n",
              "      <td>90078</td>\n",
              "      <td>hairrecipe蜂蜜保濕營養護髮精華素</td>\n",
              "      <td>[19, 1141, 11569, 1958, 5263, 5304, 5398, 6027...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26263</th>\n",
              "      <td>26264</td>\n",
              "      <td>一匙靈制菌洗衣精瓶裝</td>\n",
              "      <td>[19, 59, 23183, 1565, 299, 2440, 21816, 1191, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25412</th>\n",
              "      <td>25413</td>\n",
              "      <td>日本no1ariel日本進口三合一3d洗衣膠囊洗衣球室內晾乾六入箱</td>\n",
              "      <td>[4531, 4615, 41, 8816, 1725, 161, 10046, 86, 1...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37753</th>\n",
              "      <td>37754</td>\n",
              "      <td>cnplaboratory蜂膠能量彈潤乳霜</td>\n",
              "      <td>[19, 4447, 1116, 2453, 18960, 14707, 592, 5398...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89670</th>\n",
              "      <td>89671</td>\n",
              "      <td>炫豔護髮染髮乳自然褐白染</td>\n",
              "      <td>[19, 17141, 21748, 2267, 4532, 6342, 4532, 325...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-67f6faac-255e-47e2-83f5-dcfd99b1464a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-67f6faac-255e-47e2-83f5-dcfd99b1464a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-67f6faac-255e-47e2-83f5-dcfd99b1464a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uRpKDj-WN2tN"
      },
      "source": [
        "## **3. Train, Valid Split**\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a1SXXP4GN1m_"
      },
      "source": [
        "# train valid split\n",
        "train, valid = train_test_split(train, test_size=0.2, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H2A1pqzcN_Gr"
      },
      "source": [
        "X_train = train[\"features\"].values.tolist()\n",
        "X_valid = valid[\"features\"].values.tolist()\n",
        "\n",
        "train_masks = train[\"masks\"].values.tolist()\n",
        "valid_masks = valid[\"masks\"].values.tolist()\n",
        "\n",
        "# products column names\n",
        "label_cols = products.columns.values.tolist()\n",
        "Y_train = train[label_cols].values.tolist()\n",
        "Y_valid = valid[label_cols].values.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GQUZkER-OCVE"
      },
      "source": [
        "## **4. Create Dataloaders (torch tensors)**\n",
        "---\n",
        "\n",
        "Convert all of the input ids and attention masks into torch tensors, the required datatype for the model.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jChp-AsjOBrv"
      },
      "source": [
        "X_train = torch.tensor(X_train)\n",
        "X_valid = torch.tensor(X_valid)\n",
        "\n",
        "Y_train = torch.tensor(Y_train, dtype=torch.float32)\n",
        "Y_valid = torch.tensor(Y_valid, dtype=torch.float32)\n",
        "\n",
        "train_masks = torch.tensor(train_masks, dtype=torch.long)\n",
        "valid_masks = torch.tensor(valid_masks, dtype=torch.long)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KMeaH6gbOHM1"
      },
      "source": [
        "# Select a batch size for training\n",
        "batch_size = 16\n",
        "\n",
        "# Create an iterator of our data with torch DataLoader. \n",
        "# With an iterator, the entire dataset does not need to be loaded into memory.\n",
        "train_data = TensorDataset(X_train, train_masks, Y_train)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data,\\\n",
        "                              sampler=train_sampler,\\\n",
        "                              batch_size=batch_size)\n",
        "\n",
        "validation_data = TensorDataset(X_valid, valid_masks, Y_valid)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data,\\\n",
        "                                   sampler=validation_sampler,\\\n",
        "                                   batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **5. Training**\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ],
      "metadata": {
        "id": "7yJvXWLhesZ1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### (1) Functions"
      ],
      "metadata": {
        "id": "hrCm3qPzfDep"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Train the model and save the model with the lowest validation loss."
      ],
      "metadata": {
        "id": "STmTd2szhNjc"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XCE_3AnLOPpI"
      },
      "source": [
        "def train(model, num_epochs,\\\n",
        "          optimizer,\\\n",
        "          train_dataloader, valid_dataloader,\\\n",
        "          model_save_path,\\\n",
        "          train_loss_set=[], valid_loss_set = [],\\\n",
        "          lowest_eval_loss=None, start_epoch=0,\\\n",
        "          device=\"cpu\"\n",
        "          ):\n",
        "  model.to(device)\n",
        "\n",
        "  # trange is a tqdm wrapper around the normal python range\n",
        "  for i in trange(num_epochs, desc=\"Epoch\"):\n",
        "    # if continue training from saved model\n",
        "    actual_epoch = start_epoch + i\n",
        "    \"\"\"\n",
        "    Training\n",
        "    \"\"\"\n",
        "    # Set our model to training mode (as opposed to evaluation mode)\n",
        "    model.train()\n",
        "    # Tracking variables\n",
        "    tr_loss = 0\n",
        "    num_train_samples = 0\n",
        "\n",
        "    # Train the data for one epoch\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "      # Add batch to GPU\n",
        "      batch = tuple(t.to(device) for t in batch)\n",
        "      # Unpack the inputs from our dataloader\n",
        "      b_input_ids, b_input_mask, b_labels = batch\n",
        "      # Clear out the gradients (by default they accumulate)\n",
        "      optimizer.zero_grad()\n",
        "      # Forward pass\n",
        "      loss = model(b_input_ids, attention_mask=b_input_mask, labels=b_labels)\n",
        "      # store train loss\n",
        "      tr_loss += loss.item()\n",
        "      num_train_samples += b_labels.size(0)\n",
        "      # Backward pass\n",
        "      loss.backward()\n",
        "      # Update parameters and take a step using the computed gradient\n",
        "      optimizer.step()\n",
        "      # scheduler.step()\n",
        "\n",
        "    # Update tracking variables\n",
        "    epoch_train_loss = tr_loss/num_train_samples\n",
        "    train_loss_set.append(epoch_train_loss)\n",
        "    print(\"Train loss: {}\".format(epoch_train_loss))\n",
        "\n",
        "    \"\"\"\n",
        "    Validation\n",
        "    \"\"\"\n",
        "    # Put model in evaluation mode to evaluate loss on the validation set\n",
        "    model.eval()\n",
        "    # Tracking variables \n",
        "    eval_loss = 0\n",
        "    num_eval_samples = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in valid_dataloader:\n",
        "      # Add batch to GPU\n",
        "      batch = tuple(t.to(device) for t in batch)\n",
        "      # Unpack the inputs from our dataloader\n",
        "      b_input_ids, b_input_mask, b_labels = batch\n",
        "      # Telling the model not to compute or store gradients,\n",
        "      # saving memory and speeding up validation\n",
        "      with torch.no_grad():\n",
        "        # Forward pass, calculate validation loss\n",
        "        loss = model(b_input_ids, attention_mask=b_input_mask, labels=b_labels)\n",
        "        # store valid loss\n",
        "        eval_loss += loss.item()\n",
        "        num_eval_samples += b_labels.size(0)\n",
        "\n",
        "    # Update tracking variables\n",
        "    epoch_eval_loss = eval_loss/num_eval_samples\n",
        "    valid_loss_set.append(epoch_eval_loss)\n",
        "    print(\"Valid loss: {}\".format(epoch_eval_loss))\n",
        "\n",
        "    if lowest_eval_loss == None:\n",
        "      lowest_eval_loss = epoch_eval_loss\n",
        "      # save model\n",
        "      save_model(model, model_save_path, actual_epoch,\\\n",
        "                 lowest_eval_loss, train_loss_set, valid_loss_set)\n",
        "    else:\n",
        "      if epoch_eval_loss < lowest_eval_loss:\n",
        "        lowest_eval_loss = epoch_eval_loss\n",
        "        # save model\n",
        "        save_model(model, model_save_path, actual_epoch,\\\n",
        "                   lowest_eval_loss, train_loss_set, valid_loss_set)\n",
        "    print(\"\\n\")\n",
        "\n",
        "  return model, train_loss_set, valid_loss_set"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Save the model to the path directory provided."
      ],
      "metadata": {
        "id": "VEpPF07OhTk7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def save_model(model, save_path, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist):\n",
        "  model_to_save = model.module if hasattr(model, 'module') else model\n",
        "  checkpoint = {'epochs': epochs, \\\n",
        "                'lowest_eval_loss': lowest_eval_loss,\\\n",
        "                'state_dict': model_to_save.state_dict(),\\\n",
        "                'train_loss_hist': train_loss_hist,\\\n",
        "                'valid_loss_hist': valid_loss_hist\n",
        "               }\n",
        "  torch.save(checkpoint, save_path)\n",
        "  print(\"Saving model at epoch {} with validation loss of {}\".format(epochs,\\\n",
        "                                                                     lowest_eval_loss))\n",
        "  return"
      ],
      "metadata": {
        "id": "1D1MgfFIe6m3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load the model from the path directory provided."
      ],
      "metadata": {
        "id": "9PXzUAIAhcyC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def load_model(save_path):\n",
        "  checkpoint = torch.load(save_path)\n",
        "  model_state_dict = checkpoint['state_dict']\n",
        "  model = XLNetForMultiLabelSequenceClassification(num_labels=model_state_dict[\"classifier.weight\"].size()[0])\n",
        "  model.load_state_dict(model_state_dict)\n",
        "\n",
        "  epochs = checkpoint[\"epochs\"]\n",
        "  lowest_eval_loss = checkpoint[\"lowest_eval_loss\"]\n",
        "  train_loss_hist = checkpoint[\"train_loss_hist\"]\n",
        "  valid_loss_hist = checkpoint[\"valid_loss_hist\"]\n",
        "  \n",
        "  return model, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist"
      ],
      "metadata": {
        "id": "YGzpUwd6haZc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OoxSrbFDOUW0"
      },
      "source": [
        "### (2) Train model from scratch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wGMnhaIuUbXx"
      },
      "source": [
        "torch.cuda.empty_cache()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8VMI7VWpOKj0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121,
          "referenced_widgets": [
            "dd941141a74240a0b511215e71ae98d3",
            "9e1a945e631c49edbdfd4b69af8cbef0",
            "b3da2ad6362d49c6971f450b65c9b672",
            "de52d1d0781a4e028a4119ef5915d353",
            "ecc6276356734878bd2480e18da25587",
            "6bf46a63168f429a879848946dc97c41",
            "25c35d71e96240a18cb7e02189ef9bd5",
            "496c2521f5304feda41ba86528a88537",
            "cd7bea6a11054cf5a19b952a0b23fa46",
            "2d7f2d4b116b41549dd13744888f571a",
            "c0b8ee64545748b49e9616028f0187e2"
          ]
        },
        "outputId": "d04c044e-fda1-4ca5-8a7d-0c3d2681f0a6"
      },
      "source": [
        "#config = XLNetConfig()\n",
        "        \n",
        "class XLNetForMultiLabelSequenceClassification(torch.nn.Module):\n",
        "  \n",
        "  def __init__(self, num_labels=2):\n",
        "    super(XLNetForMultiLabelSequenceClassification, self).__init__()\n",
        "    self.num_labels = num_labels\n",
        "    # self.xlnet = XLNetModel.from_pretrained('xlnet-base-cased')\n",
        "    self.xlnet = AutoModel.from_pretrained(\"hfl/chinese-xlnet-base\")\n",
        "    self.classifier = torch.nn.Linear(768, num_labels)\n",
        "\n",
        "    torch.nn.init.xavier_normal_(self.classifier.weight)\n",
        "\n",
        "  def forward(self, input_ids, token_type_ids=None,\\\n",
        "              attention_mask=None, labels=None):\n",
        "    # last hidden layer\n",
        "    last_hidden_state = self.xlnet(input_ids=input_ids,\\\n",
        "                                   attention_mask=attention_mask,\\\n",
        "                                   token_type_ids=token_type_ids)\n",
        "    # pool the outputs into a mean vector\n",
        "    mean_last_hidden_state = self.pool_hidden_state(last_hidden_state)\n",
        "    logits = self.classifier(mean_last_hidden_state)\n",
        "        \n",
        "    if labels is not None:\n",
        "      loss_fct = BCEWithLogitsLoss()\n",
        "      loss = loss_fct(logits.view(-1, self.num_labels),\\\n",
        "                      labels.view(-1, self.num_labels))\n",
        "      return loss\n",
        "    else:\n",
        "      return logits\n",
        "    \n",
        "  def freeze_xlnet_decoder(self):\n",
        "    \"\"\"\n",
        "    Freeze XLNet weight parameters. They will not be updated during training.\n",
        "    \"\"\"\n",
        "    for param in self.xlnet.parameters():\n",
        "      param.requires_grad = False\n",
        "    \n",
        "  def unfreeze_xlnet_decoder(self):\n",
        "    \"\"\"\n",
        "    Unfreeze XLNet weight parameters. They will be updated during training.\n",
        "    \"\"\"\n",
        "    for param in self.xlnet.parameters():\n",
        "      param.requires_grad = True\n",
        "    \n",
        "  def pool_hidden_state(self, last_hidden_state):\n",
        "    \"\"\"\n",
        "    Pool the output vectors into a single mean vector \n",
        "    \"\"\"\n",
        "    last_hidden_state = last_hidden_state[0]\n",
        "    mean_last_hidden_state = torch.mean(last_hidden_state, 1)\n",
        "    return mean_last_hidden_state\n",
        "    \n",
        "model = XLNetForMultiLabelSequenceClassification(num_labels=len(Y_train[0]))\n",
        "#model = torch.nn.DataParallel(model)\n",
        "#model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/445M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dd941141a74240a0b511215e71ae98d3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at hfl/chinese-xlnet-base were not used when initializing XLNetModel: ['lm_loss.bias', 'lm_loss.weight']\n",
            "- This IS expected if you are initializing XLNetModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLNetModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6QuSJxanOMRy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b6319bd7-a975-4f14-c8f4-84dcb46875c2"
      },
      "source": [
        "optimizer = AdamW(model.parameters(), lr=2e-5, weight_decay=0.01, correct_bias=False)\n",
        "# scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pxCN5nLvOO2a",
        "outputId": "38e00f72-e1b4-4001-a979-27c6390b247a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Need GPU\n",
        "# Modify output path\n",
        "\n",
        "num_epochs=3\n",
        "\n",
        "cwd = os.getcwd()\n",
        "model_save_path = output_model_file = os.path.join(cwd, \"/content/drive/MyDrive/碩一下/金融科技/麻布數據/output/xlnet_product.bin\")\n",
        "model, train_loss_set, valid_loss_set = train(model=model,\\\n",
        "                                              num_epochs=num_epochs,\\\n",
        "                                              optimizer=optimizer,\\\n",
        "                                              train_dataloader=train_dataloader,\\\n",
        "                                              valid_dataloader=validation_dataloader,\\\n",
        "                                              model_save_path=model_save_path,\\\n",
        "                                              device=\"cuda\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:   0%|          | 0/3 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 0.004937056757044047\n",
            "Valid loss: 0.0018341614305973054\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  33%|███▎      | 1/3 [02:25<04:50, 145.15s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving model at epoch 0 with validation loss of 0.0018341614305973054\n",
            "\n",
            "\n",
            "Train loss: 0.001713361949659884\n",
            "Valid loss: 0.001810633335262537\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  67%|██████▋   | 2/3 [04:43<02:21, 141.12s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving model at epoch 1 with validation loss of 0.001810633335262537\n",
            "\n",
            "\n",
            "Train loss: 0.0016926048835739494\n",
            "Valid loss: 0.001785643445327878\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 3/3 [07:01<00:00, 140.59s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving model at epoch 2 with validation loss of 0.001785643445327878\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e5ZPpkIVffCW",
        "outputId": "7e46d7ab-62aa-40c4-f152-8e7ec846a440",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        }
      },
      "source": [
        "# Plot loss\n",
        "num_epochs = np.arange(len(train_loss_set))\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(10, 5));\n",
        "ax.plot(num_epochs, np.array(train_loss_set), label=\"Train Loss\")\n",
        "ax.plot(num_epochs, np.array(valid_loss_set), 'g-', label=\"Valid Loss\")\n",
        "# ax1.plot(episode_record, lose_record, 'r-', label=\"Lose %\")\n",
        "ax.set_xlabel(\"Number of Epochs\")\n",
        "ax.set_ylabel(\"Loss\")\n",
        "ax.set_title(\"Loss vs Number of Epochs\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Loss vs Number of Epochs')"
            ]
          },
          "metadata": {},
          "execution_count": 37
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnQAAAFNCAYAAACJ9PI3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXhU5dnH8e89mSSA7KsICCi4ALJIgND6ttalYlVQayEoioqArXaztdW2b22tXaxttb5qZVXcWGprSzetC9raEiAoIotIBFRQ2fclIcn9/jEHHcIkBMjkzGR+n+uaKzPPeZ7n3M9MBn45M2fG3B0RERERSV+RsAsQERERkWOjQCciIiKS5hToRERERNKcAp2IiIhImlOgExEREUlzCnQiIiIiaU6BTkSklpiZm1m3kPZ9qpktMrOdZva1MGqozMzWmNl5YdchkgkU6ESkSun8H7KZnR0ErIcqtb9qZteGVFYyfQeY4+5N3P3+yhvN7GUz22dmu+IufwmhThFJAgU6EanPdgNXm1mXkOs4ImYWPYphnYGlh+lzs7s3jrtcchT7EZEUpEAnIkfMzHLN7D4z+yC43GdmucG21mb2VzPbZmZbzOzfZhYJtn3XzNYFLwuuMLNzE8w9yMw+MrOsuLbLzGxxcH2gmRWZ2Q4zW29mv6mm1G3Ao8AdVazjR2b2RNztLsFRvWhw+2Uzu8vM/nvgiJaZtTKzJ4P9L0gQFr9gZqvMbJOZ3XNg7cF815vZcjPbambPmVnnuG1uZjeZ2UpgZRX1DjWzpcF9+7KZnR60vwR8DnggqPOUau6TRPOebWZrzex7Qd1rzOyquO3NzOwxM9toZu+a2Q8qrWtssK6dZrbMzM6Mm76vmS02s+1mNtPMGgRjqvw9EZEjpyePiByN7wP5QF+gDzAQ+EGw7VvAWqAN0A74HuBmdipwMzDA3ZsAFwBrKk/s7vOIHVk7J675SuCp4Ppvgd+6e1PgZGDWYWr9KfDFYP9HowC4GugQ7G8u8AjQEljOoWHxMiAPOBMYBlwPYGbDiN0XlxO7b/4NTK809lJgENCjchFBSJsOfCMY/3fgL2aW4+7nBPMdOAL39lGs83igdbDO0cDEuPvs/4BmwEnAZ4FrgOuCur4E/ChoawoMBTbHzTscGAJ0BXoD1wbtCX9PjqJuEUGBTkSOzlXAne6+wd03Aj8mFnoA9gPtgc7uvt/d/+2xL40uB3KBHmaW7e5r3P2dKuafDowEMLMmwBf4JPzsB7qZWWt33+XuhdUV6u4fAQ8Ddx7lWh9x93fcfTvwD+Add3/B3cuA3wP9KvW/2923uPt7wH0H1gHcCPzc3ZcHY39G7OhV57ixPw/G7k1Qxwjgb+7+vLvvB34FNAQ+dQRruT84Inbg8pNK2//X3Uvc/RXgb8Dw4EhpAXC7u+909zXAr/nk8b4B+KW7L/CYYnd/N36f7v6Bu28B/kLsjwCo+vdERI6CAp2IHI0TgPj/tN8N2gDuAYqBfwYvPd4G4O7FxI4u/QjYYGYzzOwEEnsKuDx4Gfdy4LW4kDAGOAV4K3jJ8+Ia1Hs3cIGZ9anxCj+xPu763gS3G1fq/37c9fj7pTPw2wNhCtgCGLEjYonGVnbQfe7uFUH/DlWOONTX3L153OV/47ZtdffdCWpvDWRz6ON9YL+dgKqCOcBHcdf38Mn9lfD3RESOjgKdiByND4gFlANODNoIjuJ8y91PIvby2y0H3ivn7k+5+1nBWCcWtA7h7suIhYYLOfjlVtx9pbuPBNoG4582s+OqK9bdNxM7Wlb5iNRuoFHc7eOrm6eGOsVd//h+IRa+xlcKVA3d/b/xpVYz70H3uZlZsK91tVAzQItK9+OB2jcRO5pW+fE+sN/3ib0UfUSq+z0RkSOnQCcih5NtZg3iLlFiL3/+wMzamFlr4IfAEwBmdrGZdQsCx3ZiL7VWWOxz0s4JjrrtI3Z0q6Ka/T4FfB34DLGXNgnmH2VmbYIjVNuC5urmOeA3xF6ePD2ubRHwGTM70cyaAbfXYJ7DudXMWphZp6D+mUH7w8DtZtYzWEez4P1nNTULuMjMzjWzbGLvQSsB/lv9sCPyYzPLMbP/AS4Gfu/u5cG+f2pmTYKXiG8heLyBycC3zay/xXSr9DJyQlX9ntTiWkQyigKdiBzO34mFrwOXHwF3AUXAYuBN4LWgDaA78AKwi9gJBA+5+xxi75/7BbEjPh8RO8JWXYCaTuwN+C+5+6a49iHAUjPbRewEiYIq3nN2EHffAfyS2MkMB9qeJxa4FgMLgb8ebp4a+HMw1yJi70ObEuzrGWJHFGeY2Q5gCbEjkDXi7iuAUcROUNgEXAJc4u6lR1DbgbNgD1wWxm37CNhK7Kjck8CN7v5WsO2rxI5mrgJeJRa2pwZ1/Z7YiSdPATuBPxF3H1ejqt8TETkKpvegiohkNjM7G3jC3TuGXYuIHB0doRMRERFJcwp0IiIiImlOL7mKiIiIpDkdoRMRERFJcwp0IiIiImkuGnYBYWrdurV36dIl7DJEREREDmvhwoWb3L1Nom0ZHei6dOlCUVFR2GWIiIiIHJaZvVvVNr3kKiIiIpLmFOhERERE0lxSA52ZDTGzFWZWbGa3Jdiea2Yzg+3zzKxL3Lbbg/YVZnZBXPsaM3vTzBaZWVFce0sze97MVgY/WyRzbSIiIiKpImmBzsyygAeJfVdhD2CkmfWo1G0MsNXduwH3EvueQ4J+BUBPYt/b+FAw3wGfc/e+7p4X13Yb8KK7dwdeDG6LiIiI1HvJPEI3ECh291XBl0fPAIZV6jMMmBZcfxo418wsaJ/h7iXuvhooDuarTvxc04BLa2ENIiIiIikvmYGuA/B+3O21QVvCPu5eBmwHWh1mrAP/NLOFZjYurk87d/8wuP4R0K42FiEiIiKS6tLxY0vOcvd1ZtYWeN7M3nL3f8V3cHc3s4TfaRaEwHEAJ554YvKrFREREUmyZB6hWwd0irvdMWhL2MfMokAzYHN1Y939wM8NwDN88lLsejNrH8zVHtiQqCh3n+juee6e16ZNws/mExEREUkryQx0C4DuZtbVzHKIneQwu1Kf2cDo4PoVwEvu7kF7QXAWbFegOzDfzI4zsyYAZnYc8HlgSYK5RgN/TtK6RERERFJK0gJd8J64m4HngOXALHdfamZ3mtnQoNsUoJWZFQO3EJyZ6u5LgVnAMuBZ4CZ3Lyf2vrhXzewNYD7wN3d/NpjrF8D5ZrYSOC+4HarNu0qY9t81xDKqiIiISHJYJoeNvLw8T+ZXfz04p5h7nlvBuM+cxO0XnkbsBF4RERGRI2dmCyt9ZNvH0vGkiLTx5c+ezPod+5j4r1WU7C/njkt6Eoko1ImIiEjtUqBLokjE+PHQnuRGI0z692pKyyv46aVnKNSJiIhIrVKgSzIz43tfOJ3caBYPzCmmZH8Fv7yiN9EsfY2uiIiI1A4FujpgZnz7glPJjUb49fNvU1JewX0j+pKtUCciIiK1QIGuDn313O7kZkf42d/forSsggeu7EduNOvwA0VERESqoUNEdWzcZ07mx0N78vyy9Yx/fCH79peHXZKIiIikOQW6EIz+VBd+fvkZvPL2RsZMW8Ce0rKwSxIREZE0pkAXkpEDT+TXX+rD3Hc2c+3UBezctz/skkRERCRNKdCF6PIzO3L/yH4sfG8rV0+Zz/a9CnUiIiJy5BToQnZx7xN46KozWfrBdq6aXMjW3aVhlyQiIiJpRoEuBVzQ83gmXpPH2+t3MXJSIRt3loRdkoiIiKQRBboU8blT2/LItQN4d/MeCibO5aPt+8IuSURERNKEAl0K+XS31ky7fiAfbd/HiIlzWbdtb9gliYiISBpQoEsxA7u25PEbBrFldynDH57Lu5t3h12SiIiIpDgFuhR05oktmD42n92lZYyYUMg7G3eFXZKIiIikMAW6FNWrQzNmjMtnf3kFIyYU8vb6nWGXJCIiIilKgS6FnXZ8U2aOzydiUDCxkKUfbA+7JBEREUlBCnQprlvbJswaP5gG0QgjJxbyxvvbwi5JREREUowCXRro0vo4Zo4fTPNGOVw1eR5Fa7aEXZKIiIikEAW6NNGpZSNmjs+nbZNcrpk6n/++synskkRERCRFKNClkfbNGjJjfD4dWzTkukcW8MrbG8MuSURERFKAAl2aadukATPGDebkNo0ZO62IF5atD7skERERCZkCXRpqeVwO08fmc3r7Jtz4xEL+/uaHYZckIiIiIVKgS1PNGmXzxA2D6NupOTc/9Rp/en1d2CWJiIhISBTo0liTBtlMu34gg7q24puzFjFrwfthlyQiIiIhUKBLc8flRnnkugH8T/c2fOcPi3m88N2wSxIREZE6pkBXDzTIzmLSNf057/S2/O+fljD536vCLklERETqkAJdPZEbzeKhq/rzhTOO566/LefBOcVhlyQiIiJ1JBp2AVJ7cqIR7i/oR07WG9zz3ApKyir45nndMbOwSxMREZEkSuoROjMbYmYrzKzYzG5LsD3XzGYG2+eZWZe4bbcH7SvM7IJK47LM7HUz+2tc26NmttrMFgWXvslcW6qKZkX49fC+DM/ryP0vruQXz76Fu4ddloiIiCRR0o7QmVkW8CBwPrAWWGBms919WVy3McBWd+9mZgXA3cAIM+sBFAA9gROAF8zsFHcvD8Z9HVgONK2021vd/elkrSldZEWMX1zem5xohAmvrKJkfwV3XNJDR+pERETqqWQeoRsIFLv7KncvBWYAwyr1GQZMC64/DZxrsdQxDJjh7iXuvhooDubDzDoCFwGTk1h72otEjJ8M68WYs7ry6H/X8L1nllBRoSN1IiIi9VEyA10HIP6D0dYGbQn7uHsZsB1odZix9wHfASoS7POnZrbYzO41s9xjXkGaMzN+cNHpfOXsk5k+/z1ufXox5Qp1IiIi9U5aneVqZhcDG9x9YYLNtwOnAQOAlsB3q5hjnJkVmVnRxo31/8vtzYxbLziVW84/hT+8tpZvzFzE/vJEWVhERETSVTID3TqgU9ztjkFbwj5mFgWaAZurGftpYKiZrSH2Eu45ZvYEgLt/6DElwCMEL9FW5u4T3T3P3fPatGlzbCtME2bG187tzm0XnsZf3viAm596jdIyhToREZH6IpmBbgHQ3cy6mlkOsZMcZlfqMxsYHVy/AnjJY6dkzgYKgrNguwLdgfnufru7d3T3LsF8L7n7KAAzax/8NOBSYEkS15aWbvzsydxxSQ+eW7qeG59YyL795YcfJCIiIikvaWe5unuZmd0MPAdkAVPdfamZ3QkUuftsYArwuJkVA1uIhTSCfrOAZUAZcFPcGa5VedLM2gAGLAJuTMrC0tx1n+5KbjSL7//pTW6YVsSka/JomJMVdlkiIiJyDCyTP6MsLy/Pi4qKwi4jFE8vXMt3nn6DvC4tmXrtABrn6jOmRUREUpmZLXT3vETb0uqkCKk9V/TvyH0F/Vj47launjKP7Xv3h12SiIiIHCUFugw2tM8JPHjlmSxZt51Rk+exdXdp2CWJiIjIUVCgy3BDeh3PhKv7s2L9TkZOKmTTrpKwSxIREZEjpEAnnHNaO6aOHsCazbspmFjI+h37wi5JREREjoACnQBwVvfWTLtuIB9u28uICXP5YNvesEsSERGRGlKgk48NOqkVj40ZxOZdpQyfMJf3t+wJuyQRERGpAQU6OUj/zi14amw+O/eVMXzCXFZt3BV2SSIiInIYCnRyiDM6NmPGuHxKyyoYMbGQlet3hl2SiIiIVEOBThI6vX1TZo7Px4AREwtZ9sGOsEsSERGRKijQSZW6tW3CzPGDaRCNMHJSIYvXbgu7JBEREUlAgU6q1bX1ccwcP5imDaNcNWkeC9/dEnZJIiIiUokCnRxWp5aNmDV+MK2b5HL1lPkUrtocdkkiIiISR4FOaqR9s4bMHJdPh+YNufaR+fzr7Y1hlyQiIiIBBTqpsbZNGzBjXD5dWzfmhmlFvLh8fdgliYiICAp0coRaNc5l+thBnNa+CTc+sZBnl3wYdkkiIiIZT4FOjljzRjk8ccMgzujQjJueep0/L1oXdkkiIiIZTYFOjkrTBtk8NmYQeZ1b8I2Zi/h90fthlyQiIpKxFOjkqDXOjfLodQM5q1trbn16MU/OezfskkRERDKSAp0ck4Y5WUy6Jo9zT2vL959ZwtRXV4ddkoiISMZRoJNj1iA7i9+N6s+Qnsdz51+X8buX3wm7JBERkYyiQCe1Iica4YEr+zG0zwnc/exb3PfC27h72GWJiIhkhGjYBUj9Ec2KcO+IvuREI9z3wkpKyir4zgWnYmZhlyYiIlKvKdBJrcqKGL/8Ym9yoxF+9/I77Ntfzg8v7qFQJyIikkQKdFLrIhHjrkt7kRON8Mh/1lBaVsFPhvUiElGoExERSQYFOkkKM+OHF/eInTDx8juUlFVw9xd7k6VQJyIiUusU6CRpzIzvXHAqucF76krLKvjN8D5Es3QujoiISG1SoJOkMjO+cd4p5EazuPvZt9hfXsFvC/qRE1WoExERqS36X1XqxJfPPpkfXtyDfyz5iC8/sZB9+8vDLklERKTeUKCTOnP9WV2569JevPjWBsY+VsTeUoU6ERGR2pDUQGdmQ8xshZkVm9ltCbbnmtnMYPs8M+sSt+32oH2FmV1QaVyWmb1uZn+Na+sazFEczJmTzLXJ0RmV35l7rujNq8WbuO7R+ewuKQu7JBERkbSXtEBnZlnAg8CFQA9gpJn1qNRtDLDV3bsB9wJ3B2N7AAVAT2AI8FAw3wFfB5ZXmutu4N5grq3B3JKCvpTXiftG9GXBmq1cM3U+O/btD7skERGRtJbMI3QDgWJ3X+XupcAMYFilPsOAacH1p4FzLfYJtMOAGe5e4u6rgeJgPsysI3ARMPnAJMGYc4I5COa8NCmrkloxrG8HHryyH4vXbmPU5Hls21MadkkiIiJpK5mBrgPwftzttUFbwj7uXgZsB1odZux9wHeAirjtrYBtwRxV7UtSzJBe7Xl4VH/e+nAnIyfNY/OukrBLEhERSUtpdVKEmV0MbHD3hccwxzgzKzKzoo0bN9ZidXI0zj29HZNH57F60y4KJhayYce+sEsSERFJO8kMdOuATnG3OwZtCfuYWRRoBmyuZuyngaFmtobYS7jnmNkTwZjmwRxV7QsAd5/o7nnuntemTZujX53Ums+c0oZHrh3Ium17GTGxkA+37w27JBERkbSSzEC3AOgenH2aQ+wkh9mV+swGRgfXrwBecncP2guCs2C7At2B+e5+u7t3dPcuwXwvufuoYMycYA6COf+cxLVJLRt8ciseHzOQTTtLGD5hLu9v2RN2SSIiImkjaYEueD/bzcBzxM5IneXuS83sTjMbGnSbArQys2LgFuC2YOxSYBawDHgWuMndD/ehZd8FbgnmahXMLWmkf+eWPDl2EDv2ljFiwlxWb9oddkkiIiJpwWIHtzJTXl6eFxUVhV2GVLLsgx2MmjKPaMR48oZBdG/XJOySREREQmdmC909L9G2tDopQjJDjxOaMnNcPg4UTCxk+Yc7wi5JREQkpSnQSUrq3q4JM8flkxONMHJSIW+u3R52SSIiIilLgU5S1kltGjNr/GAa50a5clIhC9/dGnZJIiIiKUmBTlJap5aNmDV+MK0a53DNlHnMW7U57JJERERSjgKdpLwTmjdk1vjBtG/ekNGPzOfVlZvCLklERCSlKNBJWmjbtAEzxuXTpdVxXD9tAXPe2hB2SSIiIilDgU7SRuvGuUwfm8+p7Zow7vEinl3yUdgliYiIpAQFOkkrLY7L4YkbBnFGh2bc9NRr/OWND8IuSUREJHQKdJJ2mjXM5rExg+jfuQVfn/E6Ty9cG3ZJIiIioVKgk7TUODfKtOsG8qmTW3Pr02/w1Lz3wi5JREQkNAp0krYa5mQxeXQeZ5/Shu898yaP/md12CWJiIiEQoFO0lqD7CwmXJ3HBT3b8aO/LGPCK++EXZKIiEidU6CTtJcTjfDAlWdySZ8T+Pk/3uL+F1fi7mGXJSIiUmeiYRcgUhuysyLcN6IvOVkRfvP825SUlfPtz5+KmYVdmoiISNIp0Em9kRUx7rmiNznRCA/OeYd9+yv4wUWnK9SJiEi9p0An9UokYvzssl7kRiNMeXU1JWXl3Dm0F5GIQp2IiNRfCnRS75gZd1zSg9zsCBNeWUVpWQU/v7w3WQp1IiJSTynQSb1kZtw25DRyo1nc/+JKSssq+NWX+hDN0nlAIiJS/yjQSb1lZtxy/inkRiPc89wKSssruG9EP3KiCnUiIlK/KNBJvXfT57qRG41w19+WU1q2kAevOpPcaFbYZYmIiNQaHaqQjHDD/5zETy7txQvLNzD2sYXs218edkkiIiK1RoFOMsbV+Z355Rd78++VG7nukQXsLikLuyQREZFaoUAnGWX4gE7cO7wv89dsYfTU+ezctz/skkRERI6ZAp1knEv7deD/RvZj0fvbGDVlPtv3KNSJiEh6U6CTjPSFM9rz8Kj+LP9gByMnFbJld2nYJYmIiBw1BTrJWOf1aMek0Xm8s3EXBRPnsmHnvrBLEhEROSoKdJLRPntKGx65bgBrt+6lYEIhH21XqBMRkfSjQCcZ71Mnt+ax6weyYWcJwyfMZe3WPWGXJCIickQU6ESAvC4teeKGQWzbU8qICYWs2bQ77JJERERqLKmBzsyGmNkKMys2s9sSbM81s5nB9nlm1iVu2+1B+wozuyBoa2Bm883sDTNbamY/juv/qJmtNrNFwaVvMtcm9U/fTs2ZPi6fPaVlDJ8wl+INu8IuSUREpEaSFujMLAt4ELgQ6AGMNLMelbqNAba6ezfgXuDuYGwPoADoCQwBHgrmKwHOcfc+QF9giJnlx813q7v3DS6LkrU2qb96ntCMmeMHU+FQMHEub320I+ySREREDiuZR+gGAsXuvsrdS4EZwLBKfYYB04LrTwPnmpkF7TPcvcTdVwPFwECPOXDYJDu4eBLXIBnolHZNmDU+n2gkQsHEQpas2x52SSIiItVKZqDrALwfd3tt0Jawj7uXAduBVtWNNbMsM1sEbACed/d5cf1+amaLzexeM8utzcVIZjmpTWNmjR/McTlRRk4q5PX3toZdkoiISJXS7qQIdy93975AR2CgmfUKNt0OnAYMAFoC30003szGmVmRmRVt3LixTmqW9HRiq0bMunEwLY/LYdTkecxfvSXskkRERBJKZqBbB3SKu90xaEvYx8yiQDNgc03Guvs2YA6x99jh7h8GL8mWAI8Qe8n3EO4+0d3z3D2vTZs2R7k0yRQdmjdk1vjBHN+sAaOnzuc/xZvCLklEROQQyQx0C4DuZtbVzHKIneQwu1Kf2cDo4PoVwEvu7kF7QXAWbFegOzDfzNqYWXMAM2sInA+8FdxuH/w04FJgSRLXJhmkXdMGzBw/mM6tGnHdowuYs2JD2CWJiIgcJGmBLnhP3M3Ac8ByYJa7LzWzO81saNBtCtDKzIqBW4DbgrFLgVnAMuBZ4CZ3LwfaA3PMbDGxwPi8u/81mOtJM3sTeBNoDdyVrLVJ5mndOJfpY/M5pV1jxj1WxD+XfhR2SSIiIh+z2AGxzJSXl+dFRUVhlyFpZPve/YyeOp8l67ZzX0FfLu59QtgliYhIhjCzhe6el2hbjY7QmdlxZhYJrp9iZkPNLLs2ixRJB80aZvPEDYM488QWfG366/zxtbVhlyQiIlLjl1z/BTQwsw7AP4GrgUeTVZRIKmucG+XR6wcw+ORWfOv3bzBj/nthlyQiIhmupoHO3H0PcDnwkLt/idi3OIhkpEY5UaaMHsBnT2nDbX98k8fmrgm7JBERyWA1DnRmNhi4Cvhb0JaVnJJE0kOD7CwmXN2f83u044d/Xsqkf60KuyQREclQNQ103yD2wb3PBGeqnkTsM+BEMlpuNIuHrjqTi3q356d/X87/vbgy7JJERCQDRWvSyd1fAV4BCE6O2OTuX0tmYSLpIjsrwm9H9CU3K8Kvn3+bkrIKvvX5U4h9JKKIiEjy1fQs16fMrKmZHUfsA3uXmdmtyS1NJH1EsyL86kt9KBjQiQfmFPOzvy8nkz8SSERE6lZNX3Lt4e47iH0Dwz+ArsTOdBWRQCRi/OyyMxg9uDOT/r2aO2YvpaJCoU5ERJKvRi+5AtnB585dCjzg7vvNTP9TiVQSiRg/GtqT3OwsJv5rFaVlFfz0sjPIiujlVxERSZ6aBroJwBrgDeBfZtYZ2JGsokTSmZlx+4Wn0SAa4f6Xiiktq+CXV/QmmpXMr04WEZFMVtOTIu4H7o9retfMPpeckkTSn5lxy+dPJSca4Vf/jJ0ocV9BX7IV6kREJAlqFOjMrBlwB/CZoOkV4E5ge5LqEqkXbj6nOw2ys7jrb8spLa/ggSv7kRvVRziKiEjtqunhgqnATmB4cNkBPJKsokTqkxv+5yTuHNaT55etZ9xjC9m3vzzskkREpJ6paaA72d3vcPdVweXHwEnJLEykPrlmcBfu/uIZ/GvlRq5/dAF7SsvCLklEROqRmga6vWZ21oEbZvZpYG9yShKpn0YMOJHfDO9D4arNjJ46n5379oddkoiI1BM1Pcv1RuCx4L10AFuB0ckpSaT+uqxfR3Kysvj6jNcZNWU+j103kGaNssMuS0RE0lyNjtC5+xvu3gfoDfR2937AOUmtTKSeuqh3e343qj/LP9jBlZML2bq7NOySREQkzR3RZyi4+47gGyMAbklCPSIZ4fwe7Zh4TX+KN+yiYGIhG3eWhF2SiIiksWP5UCx99L3IMTj71LY8cu0A3tuyhxET5/LR9n1hlyQiImnqWAKdvvpL5Bh9qltrHhszkA07ShgxcS5rt+4JuyQREUlD1QY6M9tpZjsSXHYCJ9RRjSL12oAuLXl8zEC27i5lxIRC3t28O+ySREQkzVQb6Ny9ibs3TXBp4u41PUNWRA6j34kteGpsPntKyxg+YS7vbNwVdkkiIpJG9MWSIimiV4dmzBg3mPIKZ8SEQlZ8tDPskkREJE0o0ImkkFOPb8KMcYPJikDBxLksWaevSxYRkcNToBNJMd3aNmbW+ME0yoly5aRCFr2/LeySREQkxSnQiaSgzq2OY+b4fJo3ymHU5HksWLMl7JJERPXyqyoAAB87SURBVCSFKdCJpKiOLRoxa/xg2jbN5Zop8/lv8aawSxIRkRSlQCeSwo5v1oCZ4wZzYstGXPfoAl5esSHskkREJAUp0ImkuDZNcpk+Lp9ubRsz7rGFPL9sfdgliYhIiklqoDOzIWa2wsyKzey2BNtzzWxmsH2emXWJ23Z70L7CzC4I2hqY2Xwze8PMlprZj+P6dw3mKA7mzEnm2kTqUsvjcnjqhnxOP6EpX35iIX9b/GHYJYmISApJWqAzsyzgQeBCoAcw0sx6VOo2Btjq7t2Ae4G7g7E9gAKgJzAEeCiYrwQ4x937AH2BIWaWH8x1N3BvMNfWYG6ReqNZo2yeGDOQfic256vTX+OZ19eGXZKIiKSIZB6hGwgUu/sqdy8FZgDDKvUZBkwLrj8NnGtmFrTPcPcSd18NFAMDPebAR+hnBxcPxpwTzEEw56XJWphIWJo0yGba9QPJP6kVt8x6g1kL3g+7JBERSQHJDHQdgPj/bdYGbQn7uHsZsB1oVd1YM8sys0XABuB5d58XjNkWzFHVvkTqhUY5UaZeO4DPdG/Dd/6wmMfnrgm7JBERCVnanRTh7uXu3hfoCAw0s15HMt7MxplZkZkVbdy4MTlFiiRZg+wsJl7Tn/NOb8f//nkpk/+9KuySREQkRMkMdOuATnG3OwZtCfuYWRRoBmyuyVh33wbMIfYeu81A82COqvZ1YNxEd89z97w2bdocxbJEUkNuNIvfjTqTi85oz11/W86Dc4rDLklEREKSzEC3AOgenH2aQ+wkh9mV+swGRgfXrwBecncP2guCs2C7At2B+WbWxsyaA5hZQ+B84K1gzJxgDoI5/5zEtYmkhOysCL8t6Mtl/Tpwz3Mr+M0/VxB7OoiISCaJHr7L0XH3MjO7GXgOyAKmuvtSM7sTKHL32cAU4HEzKwa2EAt9BP1mAcuAMuAmdy83s/bAtOCM1wgwy93/Guzyu8AMM7sLeD2YW6Tei2ZF+NWX+pCTFeH+l4opKavgtgtPI3aukIiIZALL5L/m8/LyvKioKOwyRGpFRYVzx+ylPF74Ltd+qgt3XNJDoU5EpB4xs4XunpdoW9KO0IlI3YpEjDuH9SQ3GmHyq6spKavgp5f2IhJRqBMRqe8U6ETqETPj+xedToPsLB6YU0xJWTn3XNGHLIU6EZF6TYFOpJ4xM759wankRiP8+vm3KS2r4N4RfcnOSrtPKRIRkRpSoBOpp756bndysyP87O9vUVpWwf9d2Y/caFbYZYmISBLoT3aRemzcZ07mx0N78s9l67nx8YXs218edkkiIpIECnQi9dzoT3Xh55efwctvb+SGaUXsKS07/CAREUkrCnQiGWDkwBP51RV9+O87m7h26gJ2lSjUiYjUJwp0Ihnii/078tuCfix8bytXT5nH9r37wy5JRERqiQKdSAa5pM8JPHTVmSxZt52rJheydXdp2CWJiEgtUKATyTAX9DyeiVfn8fb6XYycVMimXSVhlyQiIsdIgU4kA33utLY8cu0A1mzezYgJc1m/Y1/YJYmIyDFQoBPJUJ/u1prHrh/ER9v3MWLCXNZt2xt2SSIicpQU6EQy2MCuLXn8hkFs3l3KiAlzeX/LnrBLEhGRo6BAJ5LhzjyxBdPH5rOrpIwvPTyXVRt3hV2SiIgcIQU6EaFXh2bMGJfP/vIKhk8o5O31O8MuSUREjoACnYgAcNrxTZk5Pp+IQcHEQpZ+sD3skkREpIYU6ETkY93aNmHW+ME0iEa4ctI83nh/W9gliYhIDSjQichBurQ+jpnjB9OsYTajJs+jaM2WsEsSEZHDUKATkUN0atmImePzadMkl2umzmfuO5vDLklERKqhQCciCbVv1pAZ4/Pp0Lwh1z4yn3+9vTHskkREpAoKdCJSpbZNGjBjXD4nt2nMDdOKeHH5+rBLEhGRBBToRKRarRrn8tTYQZzevgnjH1/IP978MOySRESkEgU6ETms5o1yePyGQfTp1Jybp7/OnxetC7skERGJo0AnIjXStEE2j10/kAFdWvCNmYuYVfR+2CWJiEhAgU5Eauy43CiPXDuQs7q15jtPL+aJwnfDLklERFCgE5Ej1DAni0nX5HHe6W35wZ+WMOXV1WGXJCKS8RToROSINcjO4qGr+nNhr+P5yV+X8dDLxWGXJCKS0RToROSo5EQj/N/IfgzrewK/fHYF9z7/Nu4edlkiIhkpGnYBIpK+olkRfjO8L7nRCL99cSUlZRV8d8ipmFnYpYmIZJSkHqEzsyFmtsLMis3stgTbc81sZrB9npl1idt2e9C+wswuCNo6mdkcM1tmZkvN7Otx/X9kZuvMbFFw+UIy1yYiMVkR4xeX92ZU/ok8/Mo73PnXZTpSJyJSx5J2hM7MsoAHgfOBtcACM5vt7sviuo0Btrp7NzMrAO4GRphZD6AA6AmcALxgZqcAZcC33P01M2sCLDSz5+PmvNfdf5WsNYlIYpGI8ZNhvciNZjHl1dWUlFVw17BeRCI6UiciUheSeYRuIFDs7qvcvRSYAQyr1GcYMC24/jRwrsVeqxkGzHD3EndfDRQDA939Q3d/DcDddwLLgQ5JXIOI1JCZ8YOLTucrZ5/MU/Pe4zt/WEx5hY7UiYjUhWQGug5A/CePruXQ8PVxH3cvA7YDrWoyNnh5th8wL675ZjNbbGZTzazFsS9BRI6EmXHrBadyy/mn8PTCtXxz5iL2l1eEXZaISL2Xlme5mllj4A/AN9x9R9D8O+BkoC/wIfDrKsaOM7MiMyvauHFjndQrkknMjK+d253bLjyN2W98wFefep3SMoU6EZFkSmagWwd0irvdMWhL2MfMokAzYHN1Y80sm1iYe9Ld/3igg7uvd/dyd68AJhF7yfcQ7j7R3fPcPa9NmzbHsDwRqc6Nnz2ZOy7pwbNLP+LGJxayb3952CWJiNRbyQx0C4DuZtbVzHKIneQwu1Kf2cDo4PoVwEseOz1uNlAQnAXbFegOzA/eXzcFWO7uv4mfyMzax928DFhS6ysSkSNy3ae78rPLzmDOig2MfayIvaUKdSIiyZC0QBe8J+5m4DliJy/McvelZnanmQ0Nuk0BWplZMXALcFswdikwC1gGPAvc5O7lwKeBq4FzEnw8yS/N7E0zWwx8DvhmstYmIjV35aATueeKPvyneBPXPjKfXSVlYZckIlLvWCZ/XlReXp4XFRWFXYZIRpj9xgd8c+Yi+nRsxqPXD6Rpg+ywSxIRSStmttDd8xJtS8uTIkQk/QztcwIPXnkmb67bzqjJ89i2pzTskkRE6g0FOhGpM0N6Hc+Eq/vz1kc7KZhYyOZdJWGXJCJSLyjQiUidOue0dkwZnceazbsZMbGQDTv2hV2SiEjaU6ATkTr3P93b8Oh1A/lw216GT5jLB9v2hl2SiEhaU6ATkVDkn9SKx8YMYvOuUoZPmMv7W/aEXZKISNpSoBOR0PTv3IInxw5i574yhk+Yy+pNu8MuSUQkLSnQiUioendszvSx+ZSWVTB8wlxWrt8ZdkkiImlHgU5EQtfjhKbMGJePAQUTC1n2wY7DjhERkU8o0IlISujergkzxw8mJxph5KRCFq/dFnZJIiJpQ4FORFJG19bHMWv8YJo2jHLVpHksfHdr2CWJiKQFBToRSSmdWjZi5rjBtG6Sy9VT5lG4anPYJYmIpDwFOhFJOSc0b8jMcfl0aN6Qax+Zz79Xbgy7JBGRlKZAJyIpqW3TBswYl0/X1o0ZM62Il95aH3ZJIiIpS4FORFJWq8a5TB87iNOOb8L4xxfy7JKPwi5JRCQlKdCJSEpr3iiHJ24YxBkdmnHTU68x+40Pwi5JRCTlKNCJSMpr2iCbx8YMIq9zC74x43WeXrg27JJERFKKAp2IpIXGuVEevW4gn+7Wmm///g2emvde2CWJiKQMBToRSRsNc7KYdE0e557Wlu898yaP/Gd12CWJiKQEBToRSSsNsrP43aj+DOl5PD/+yzIefuWdsEsSEQmdAp2IpJ2caIQHruzH0D4n8It/vMVvX1iJu4ddlohIaKJhFyAicjSiWRHuHdGXnGiEe194m5Kycm694FTMLOzSRETqnAKdiKStrIjxyy/2Jica4aGX36GkrIIfXHS6Qp2IZBwFOhFJa5GI8dNLe5EbjTDl1dWUlJVz59BeRCIKdSKSORToRCTtmRk/vLgHudEsHn7lHUrLKvj55b3JUqgTkQyhQCci9YKZ8d0hp9IgO8J9L6ykpKyCX3+pD9EsnfslIvWfAp2I1BtmxjfOO4WcaIRfPruC0rIKflvQj5yoQp2I1G/6V05E6p2vnN2N/724B/9Y8hFfeXIh+/aXh12SiEhSKdCJSL005qyu3HVpL15YvoGxjxWxt1ShTkTqLwU6Eam3RuV35pdX9ObV4k1c9+h8dpeUhV2SiEhSJDXQmdkQM1thZsVmdluC7blmNjPYPs/MusRtuz1oX2FmFwRtncxsjpktM7OlZvb1uP4tzex5M1sZ/GyRzLWJSHoYnteJ+0b0ZcGarVwzdT479u0PuyQRkVqXtEBnZlnAg8CFQA9gpJn1qNRtDLDV3bsB9wJ3B2N7AAVAT2AI8FAwXxnwLXfvAeQDN8XNeRvwort3B14MbouIMKxvBx4Y2Y/Fa7dx9eR5bNtTGnZJIiK1KplH6AYCxe6+yt1LgRnAsEp9hgHTgutPA+da7CPehwEz3L3E3VcDxcBAd//Q3V8DcPedwHKgQ4K5pgGXJmldIpKGLjyjPQ+P6s/yD3dy5aR5bN5VEnZJIiK1JpmBrgPwftzttXwSvg7p4+5lwHagVU3GBi/P9gPmBU3t3P3D4PpHQLtjXYCI1C/nnt6OyaPzWLVpFwUTC9mwc1/YJYmI1Iq0PCnCzBoDfwC+4e47Km93dwe8irHjzKzIzIo2btyY5EpFJNV85pQ2PHLtQNZt20vBhEI+3L437JJERI5ZMgPdOqBT3O2OQVvCPmYWBZoBm6sba2bZxMLck+7+x7g+682sfdCnPbAhUVHuPtHd89w9r02bNke5NBFJZ4NPbsXjYwaycWcJwyfM5f0te8IuSUTkmCQz0C0AuptZVzPLIXaSw+xKfWYDo4PrVwAvBUfXZgMFwVmwXYHuwPzg/XVTgOXu/ptq5hoN/LnWVyQi9Ub/zi15cuwgduwtY8SEuazZtDvskkREjlrSAl3wnribgeeInbwwy92XmtmdZjY06DYFaGVmxcAtBGemuvtSYBawDHgWuMndy4FPA1cD55jZouDyhWCuXwDnm9lK4LzgtohIlXp3bM70sfnsK6tg+IS5FG/YGXZJIiJHxWIHxDJTXl6eFxUVhV2GiIRs5fqdXDl5HhUVzhM3DOL09k3DLklE5BBmttDd8xJtS8uTIkREalP3dk2YOS6f7KwIIycVsmTd9rBLEhE5Igp0IiLASW0aM2v8YBrnRhk5qZDX3tsadkkiIjWmQCciEjixVSNmjh9Mq+NyuHryPOav3hJ2SSIiNaJAJyISp0PzhswcP5jjmzVg9NT5/Kd4U9gliYgclgKdiEgl7Zo2YOb4wXRu1YjrHl3AnLcSfqyliEjKUKATEUmgdeNcpo/N59R2TRj3eBHPLf0o7JJERKqkQJdEe/bvYcPuDWzas4kte7ewbd82tu/bzs6Snewu3c3e/XspKSuhtLyUsooyKryCTP4YGZFU0+K4HJ64YRC9OjTjK0++xl/e+CDskkREEoqGXUB9NnHhRL753DePeJxhmBkRixx0MRK0Jeh3JH3rdE7CqT1Rv3SZs6b3UexLVCQZmjXM5vExg7j+0QV8fcbrlJZV8MX+HcMuS0TkIAp0SfS5Lp/jwS88SIVXHHRx90PbSNBWw36H9KV293Pg6GEyak/Ur6q+Ur2UDPG1PWeItZ/f31lb8i43/nEOL7/fmbO6tT3i/Vd3qWqeascc4T4U/kXqL31ThL4pIm24+yFB76hDbx2H0VSbs8r7qK72U4dzyqFSOWwe0RjqpraUXX8t7efAq0KS+qr7pggdoZO0YWYf/+MlUlOVg96xBseSsnLu/MubvFq8ibGf6cwXz+x4UN/yivKPb1cV0Ku7VLXfascc4X6OaR/UTl3xR/7rev0K+4c6XDCsV8E5SWH7nK7ncFKLk0J7DBXoRKReMzOyLIsssmptzieu7c4ts97g0Vc+oFVOA756bvdam1vqTp0G4RTaR63vp5qQfyS1lVeUs9/3h3o/H4uZV8xUoBMRSSfZWRHuG9GXnKwIv37+bUrKKvjW50/Ry1ZpJhlhX9JbdcHwcEGzZcOWodauQCcichSyIsY9V/QmJxrhgTnF7NtfzvcvOl2hTiSNpXPIV6ATETlKkYjxs8t6kRuNMPnV1ZSUVfDjoT2JRBTqRKRuKdCJiBwDM+OOS3qQmx1hwiurKC2r4GeXn0GWQp2I1CEFOhGRY2Rm3DbkNHKjWdz/4kpefGsDudEIZmAGETMiZhgEbUYkaIfYz0/6xTpFDIxPxmIEbUYkUs1YgrH2yTaL238kuP3xdir346BtH4+1Go6N279Vqi328RgHr//jfsT1q9wWqWIsCfZRaexB98PHtdlh1lLd/VrpsYscXDvV3q8H3w+RSvs/0FfkaCjQiYjUAjPjlvNPoWPzhixYswUHKtzBYz8rYlcrtTnuUOEAsT6ftMU+VqPCnYoKgjdkA+7sL/eP5zrQFps/1jd+rMfN6Yna4scGtz/eDlRUVDM2bkyFPgWkVlhckE8clKsKg1WE8cihY+P/UDgoAEeqCcocHHYPDeNVBPQEYfyT9cX9oRD8oXL4MF7F2EQBPZJ4LMSt6+MgfvAfCp/8EVXV2E/2fyDs9+7YnOObNajT35d4CnQiIrVo+IBODB/QKewyQnFIGPw4NFZqqzg0DDoHh8Uqx1YOoRVVjCWuX00DbHVjP277JIQfGOuVwrjHheEDQT5+P/HrPbCfQ8bG7f/Q+RKMTbSPhPPF1VyReGysf+L7pqICyqk4+LGr9Pgk/MOgqseuqrEk+l2o/o8PD/mPigeu7MfFvU8Ibf8KdCIiUis+PkKCXjaUcFQOyAcd6Y4/mhx/lJz4I+cHH51OODbB0XR36NSiUUirjlGgExERkXoh9rEjkJWBf1ToO5RERERE0pwCnYiIiEiaU6ATERERSXMKdCIiIiJpToFOREREJM0p0ImIiIikOQU6ERERkTSnQCciIiKS5hToRERERNKcAp2IiIhImjMP+9tsQ2RmG4F3k7yb1sCmJO8jlWXy+jN57ZDZ69faM1cmrz+T1w51s/7O7t4m0YaMDnR1wcyK3D0v7DrCksnrz+S1Q2avX2vPzLVDZq8/k9cO4a9fL7mKiIiIpDkFOhEREZE0p0CXfBPDLiBkmbz+TF47ZPb6tfbMlcnrz+S1Q8jr13voRERERNKcjtCJiIiIpDkFumNgZkPMbIWZFZvZbQm255rZzGD7PDPrErft9qB9hZldUJd114YarP0WM1tmZovN7EUz6xy3rdzMFgWX2XVbee2owfqvNbONceu8IW7baDNbGVxG123lx64Ga783bt1vm9m2uG1p/dib2VQz22BmS6rYbmZ2f3DfLDazM+O2pfvjfri1XxWs+U0z+6+Z9YnbtiZoX2RmRXVXde2pwfrPNrPtcb/fP4zbVu1zJtXVYO23xq17SfA8bxlsS+vH3sw6mdmc4P+zpWb29QR9UuN57+66HMUFyALeAU4CcoA3gB6V+nwFeDi4XgDMDK73CPrnAl2DebLCXlMtr/1zQKPg+pcPrD24vSvsNdTB+q8FHkgwtiWwKvjZIrjeIuw11ebaK/X/KjC1Hj32nwHOBJZUsf0LwD8AA/KBefXhca/h2j91YE3AhQfWHtxeA7QOew1JXv/ZwF8TtB/RcyYVL4dbe6W+lwAv1ZfHHmgPnBlcbwK8neDf+5R43usI3dEbCBS7+yp3LwVmAMMq9RkGTAuuPw2ca2YWtM9w9xJ3Xw0UB/Oli8Ou3d3nuPue4GYh0LGOa0ymmjz2VbkAeN7dt7j7VuB5YEiS6kyGI137SGB6nVRWB9z9X8CWaroMAx7zmEKguZm1J/0f98Ou3d3/G6wN6t9zviaPfVWO5d+LlHCEa69vz/kP3f214PpOYDnQoVK3lHjeK9AdvQ7A+3G313Log/xxH3cvA7YDrWo4NpUdaf1jiP31ckADMysys0IzuzQZBSZZTdf/xeDw+9Nm1ukIx6aqGtcfvMzeFXgprjndH/vDqer+SffH/UhVfs478E8zW2hm40KqqS4MNrM3zOwfZtYzaMuYx97MGhELLH+Ia643j73F3jbVD5hXaVNKPO+jyZpYBMDMRgF5wGfjmju7+zozOwl4yczedPd3wqkwaf4CTHf3EjMbT+xI7Tkh11TXCoCn3b08ri0THvuMZmafIxbozoprPit43NsCz5vZW8FRn/rkNWK/37vM7AvAn4DuIddU1y4B/uPu8Ufz6sVjb2aNiQXVb7j7jrDrSURH6I7eOqBT3O2OQVvCPmYWBZoBm2s4NpXVqH4zOw/4PjDU3UsOtLv7uuDnKuBlYn/xpJPDrt/dN8eteTLQv6ZjU9yR1F9ApZde6sFjfzhV3T/p/rjXiJn1Jvb7PszdNx9oj3vcNwDPkF5vMakRd9/h7ruC638Hss2sNRny2Aeqe86n7WNvZtnEwtyT7v7HBF1S43lf128wrC8XYkc3VxF7SenAG117VupzEwefFDEruN6Tg0+KWEV6nRRRk7X3I/ZG4O6V2lsAucH11sBK0u8NwjVZf/u465cBhcH1lsDq4H5oEVxvGfaaanPtQb/TiL0Z2urTYx/U3oWq3xh/EQe/OXp+fXjca7j2E4m9H/hTldqPA5rEXf8vMCTstSRh/ccf+H0nFlreC34PavScSfVLdWsPtjcj9j674+rTYx88ho8B91XTJyWe93rJ9Si5e5mZ3Qw8R+wspqnuvtTM7gSK3H02MAV43MyKif2iFwRjl5rZLGAZUAbc5Ae/LJXSarj2e4DGwO9j54HwnrsPBU4HJphZBbEjxL9w92WhLOQo1XD9XzOzocQe3y3EznrF3beY2U+ABcF0d/rBL0+ktBquHWK/6zM8+FctkPaPvZlNJ3Y2Y2szWwvcAWQDuPvDwN+JnfFWDOwBrgu2pfXjDjVa+w+JvUf4oeA5X+axLypvBzwTtEWBp9z92TpfwDGqwfqvAL5sZmXAXqAg+P1P+JwJYQlHrQZrh9gfrv90991xQ+vDY/9p4GrgTTNbFLR9j9gfMCn1vNc3RYiIiIikOb2HTkRERCTNKdCJiIiIpDkFOhEREZE0p0AnIiIikuYU6ERERETSnAKdiKQ0M3Mz+3Xc7W+b2Y9qae5HzeyK2pjrMPv5kpktN7M5ldq7mNleM1sUd7mmFvd7tpn9tbbmE5HUpc+hE5FUVwJcbmY/d/dNYRdzgJlFPfYdzTUxBhjr7q8m2PaOu/etxdJEJAPpCJ2IpLoyYCLwzcobKh9hM7Ndwc+zzewVM/uzma0ys1+Y2VVmNt/M3jSzk+OmOc/MiszsbTO7OBifZWb3mNkCM1scfB/vgXn/bWaziX0weOV6RgbzLzGzu4O2HxL7XtMpZnZPTRdtZrvM7F4zW2pmL5pZm6C9r5kVBnU9Y2YtgvZuZvZC8OXwr8WtsbGZPW1mb5nZkxZ8ymtwnywL5vlVTesSkdSkQCci6eBB4Coza3YEY/oANxL7hoqrgVPcfSCx7xr9aly/LsS+quki4GEza0DsiNp2dx8ADADGmlnXoP+ZwNfd/ZT4nZnZCcDdwDlAX2CAmV3q7ncCRcBV7n5rgjpPrvSS6/+3dz8hNoVhHMe/z0g0f1DsrIxIpgzJLPwZOytpUpOoITsKyYaFlWyVlIVIKbMwKc2ChQ03NkaJZiWJNDVlFhr/mgn3Z/G+t46JcedKc0/9PnXrnOfee857z+L29LzvOc+OHG8jdd/oAiqkp/NDakN0WtIGYLQQHwQuS+oGtgLjOb4JOAmsBzqBbRGxnPRk/658nPN/u5hm1tyc0JlZ05P0kZTInJjD155KGpc0TeorfD/HR0lJXM2QpKqkV6Sem+uAXcDB3OrnCaml1Zr8+RFJb35zvi3AQ0kTeSp2EOitY5yvJW0svB7leBW4lbdvAttzQrtMUiXHbwC9EdEBrJR0B0DSlKSvhfGOSaoCz/NvnwSmSFXDvaR2RWZWYk7ozKwsLpIqZ22F2Hfy/1hEtJCan9dMF7arhf0qv64fntn/UKQm28cLSdYqSbWE8Avzo9E+jcXr8AOorf3rAW4Du4Gy9dc0sxmc0JlZKeSm1kOkpK7mLbA5b+8hNwyfo/6IaMlrzjqBl6RG6kcjYiFARKyNiLbZDgKMADsjYkVELAD2k6ZKG9VCavgOcAB4LGkS+FCYlh0AKpI+AWMR0ZfHuygiWv904IhoB5ZKukdam9j9D+M0sybgu1zNrEwuAMcK+1eB4Yh4QaoyNVI9e0dKxpYARyRNRcQ10tTks3wTwQTQN9tBJI1HxBngAanCd1fScB3nX52ndmuuS7pE+i09EXEWeA/sy+8fIq31ayVNER/O8QHgSkScA74B/bOcs4N03RbnsZ6qY5xm1sRCarSKb2Zm/0tEfJbUPt/jMLNy8JSrmZmZWcm5QmdmZmZWcq7QmZmZmZWcEzozMzOzknNCZ2ZmZlZyTujMzMzMSs4JnZmZmVnJOaEzMzMzK7mft8ezCFmet14AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sHt9MdUoOaLl"
      },
      "source": [
        "### (3) Train model from previous checkpoint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lzZEnoI3Ob2u",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "66088c3e-2286-4b77-8b39-dec2dfd920cd"
      },
      "source": [
        "cwd = os.getcwd()\n",
        "model_save_path = output_model_file = os.path.join(cwd, \"/content/drive/MyDrive/碩一下/金融科技/麻布數據/output/xlnet_product.bin\")\n",
        "model, start_epoch, lowest_eval_loss, train_loss_hist, valid_loss_hist = load_model(model_save_path)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at hfl/chinese-xlnet-base were not used when initializing XLNetModel: ['lm_loss.bias', 'lm_loss.weight']\n",
            "- This IS expected if you are initializing XLNetModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLNetModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HrTdby8Uaqqb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "122dce7f-6568-4338-edd4-8510f0b1ad73"
      },
      "source": [
        "optimizer = AdamW(model.parameters(), lr=2e-5, weight_decay=0.01, correct_bias=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiwYWrPsREQ-",
        "outputId": "a617afb8-9bd2-42de-e136-2cc367e1fe77",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "num_epochs=3\n",
        "model, train_loss_set, valid_loss_set = train(model=model,\\\n",
        "                                              num_epochs=num_epochs,\\\n",
        "                                              optimizer=optimizer,\\\n",
        "                                              train_dataloader=train_dataloader,\\\n",
        "                                              valid_dataloader=validation_dataloader,\\\n",
        "                                              model_save_path=model_save_path,\\\n",
        "                                              train_loss_set=train_loss_hist,\\\n",
        "                                              valid_loss_set=valid_loss_hist,\\\n",
        "                                              lowest_eval_loss=lowest_eval_loss,\\\n",
        "                                              start_epoch=start_epoch,\\\n",
        "                                              device=\"cuda\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:   0%|          | 0/3 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 0.001779610172379762\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  33%|███▎      | 1/3 [02:16<04:33, 136.62s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Valid loss: 0.001828791666775942\n",
            "\n",
            "\n",
            "Train loss: 0.001609007737133652\n",
            "Valid loss: 0.001654445333406329\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  67%|██████▋   | 2/3 [04:35<02:17, 137.69s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving model at epoch 3 with validation loss of 0.001654445333406329\n",
            "\n",
            "\n",
            "Train loss: 0.0012759967183228582\n",
            "Valid loss: 0.0012926842737942933\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 3/3 [06:53<00:00, 137.83s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving model at epoch 4 with validation loss of 0.0012926842737942933\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cew4rYj16aJl",
        "outputId": "1e19ae9c-5f6a-49e1-936c-d197104f582b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        }
      },
      "source": [
        "# Plot loss\n",
        "num_epochs = np.arange(len(train_loss_set))\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(10, 5));\n",
        "ax.plot(num_epochs, np.array(train_loss_set), label=\"Train Loss\")\n",
        "ax.plot(num_epochs, np.array(valid_loss_set), 'g-', label=\"Valid Loss\")\n",
        "# ax1.plot(episode_record, lose_record, 'r-', label=\"Lose %\")\n",
        "ax.set_xlabel(\"Number of Epochs\")\n",
        "ax.set_ylabel(\"Loss\")\n",
        "ax.set_title(\"Loss vs Number of Epochs\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Loss vs Number of Epochs')"
            ]
          },
          "metadata": {},
          "execution_count": 41
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnQAAAFNCAYAAACJ9PI3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3yU9Z33/9dnZnKAkIRTOIWEQ0AtnlAjHhCwuq3YdsV2e8CebGuX4q2/bbftbnXvve9tu9t71713a3fvbaWs2trWFu1py9rW1lYFREXCQQQUDadAOEMSCOQ8n98fcwUmIQkBmVwzyfv5eMxjrrmu73XN55pBefP9Xt9rzN0RERERkcwVCbsAEREREXl7FOhEREREMpwCnYiIiEiGU6ATERERyXAKdCIiIiIZToFOREREJMMp0ImInCdm5mY2JaT3vtDM1pvZMTP7izBq6MzMdpjZn4Rdh8hAoEAnIt3K5L+QzezGIGB9p9P6F8zsUyGVlUp/DTzn7vnu/u+dN5rZ82bWaGb1SY//DqFOEUkBBToR6c+OA58ws4kh13FWzCx2DrtNADadoc297j4k6fGn5/A+IpKGFOhE5KyZWY6ZfcvM9gSPb5lZTrBtpJk9ZWa1ZnbEzFaYWSTY9hUzqw6GBbeY2c1dHPsaM9tnZtGkde83sw3B8gwzqzCzo2a238y+2UOptcD3gb/r5jy+amY/Sno9MejViwWvnzezfzCzF9t7tMxshJk9Hrz/6i7C4nvMbJuZHTKz/9t+7sHxPmNmr5tZjZn9zswmJG1zM7vHzN4C3uqm3tvMbFPw2T5vZu8I1j8LvBP4j6DOC3r4TLo67o1mttvM/iaoe4eZfSxpe6GZ/cDMDprZTjP7207n9efBeR0zs81mdmXS4aeb2QYzqzOzJ8wsN9in2z8nInL29B+PiJyL/wlcC0wHLgdmAH8bbPsSsBsoAkYDfwO4mV0I3Atc7e75wC3Ajs4HdvdVJHrWbkpa/VHgx8HyvwH/5u4FQBnw5Blq/QbwZ8H7n4v5wCeA4uD9XgK+BwwHXuf0sPh+oBy4EpgHfAbAzOaR+Cw+QOKzWQH8pNO+twPXANM6FxGEtJ8AXwj2/w3w32aW7e43Bcdr74F78xzOcwwwMjjPO4HFSZ/Z/wMKgcnAHOCTwKeDuj4EfDVYVwDcBhxOOu6HgbnAJOAy4FPB+i7/nJxD3SKCAp2InJuPAV939wPufhD4GonQA9ACjAUmuHuLu6/wxI9GtwE5wDQzy3L3He6+tZvj/wS4A8DM8oH3cCr8tABTzGyku9e7+8s9Feru+4BFwNfP8Vy/5+5b3b0O+C2w1d3/4O6twE+BKzq1f8Ddj7h7FfCt9vMAFgL/6O6vB/v+HxK9VxOS9v3HYN+GLur4CPBrd3/G3VuAfwEGAdefxbn8e9Aj1v74+07b/5e7N7n7MuDXwIeDntL5wP3ufszddwD/yqnv+7PAP7v7ak+odPedye/p7nvc/Qjw3yT+EQDd/zkRkXOgQCci52IckPyX9s5gHcD/BSqB3wdDj/cBuHslid6lrwIHzGyJmY2jaz8GPhAM434AWJsUEu4CLgDeCIY839eLeh8AbjGzy3t9hqfsT1pu6OL1kE7tdyUtJ38uE4B/aw9TwBHASPSIdbVvZx0+c3ePB+2Lu93jdH/h7kOTHv8raVuNux/vovaRQBanf9/t71sCdBfMAfYlLZ/g1OfV5Z8TETk3CnQici72kAgo7UqDdQS9OF9y98kkht++2H6tnLv/2N1vCPZ1EkHrNO6+mURouJWOw624+1vufgcwKtj/Z2aW11Ox7n6YRG9Z5x6p48DgpNdjejpOL5UkLZ/8XEiEr891ClSD3P3F5FJ7OG6Hz9zMLHiv6vNQM8CwTp9je+2HSPSmdf6+2993F4mh6LPS058TETl7CnQiciZZZpab9IiRGP78WzMrMrORwP8GfgRgZu8zsylB4KgjMdQat8R90m4Ket0aSfRuxXt43x8DnwdmkxjaJDj+x82sKOihqg1W93Scdt8kMTz5jqR164HZZlZqZoXA/b04zpn8lZkNM7OSoP4ngvWLgPvN7OLgPAqD689660ngvWZ2s5llkbgGrQl4sefdzsrXzCzbzGYB7wN+6u5twXt/w8zygyHiLxJ838DDwJfN7CpLmNJpGLlL3f05OY/nIjKgKNCJyJn8hkT4an98FfgHoALYALwGrA3WAUwF/gDUk5hA8B13f47E9XP/RKLHZx+JHraeAtRPSFyA/6y7H0paPxfYZGb1JCZIzO/mmrMO3P0o8M8kJjO0r3uGRODaAKwBnjrTcXrhV8Gx1pO4Du2R4L1+SaJHcYmZHQU2kuiB7BV33wJ8nMQEhUPAnwJ/6u7NZ1Fb+yzY9seapG37gBoSvXKPAwvd/Y1g2/9HojdzG/ACibD9aFDXT0lMPPkxcAz4L5I+4x509+dERM6B6RpUEZGBzcxuBH7k7uPDrkVEzo166EREREQynAKdiIiISIbTkKuIiIhIhlMPnYiIiEiGU6ATERERyXCxsAsI08iRI33ixIlhlyEiIiJyRmvWrDnk7kVdbRvQgW7ixIlUVFSEXYaIiIjIGZnZzu62achVREREJMMp0ImIiIhkuJQGOjOba2ZbzKzSzO7rYnuOmT0RbF9lZhOTtt0frN9iZrckrd9hZq+Z2Xozq0haP9zMnjGzt4LnYak8NxEREZF0kbJAZ2ZR4NskfqtwGnCHmU3r1OwuoMbdpwAPkvidQ4J284GLSfxu43eC47V7p7tPd/fypHX3AX9096nAH4PXIiIiIv1eKnvoZgCV7r4t+PHoJcC8Tm3mAY8Fyz8DbjYzC9Yvcfcmd98OVAbH60nysR4Dbj8P5yAiIiKS9lIZ6IqBXUmvdwfrumzj7q1AHTDiDPs68HszW2NmC5LajHb3vcHyPmD0+TgJERERkXSXibctucHdq81sFPCMmb3h7suTG7i7m1mXv2kWhMAFAKWlpamvVkRERCTFUtlDVw2UJL0eH6zrso2ZxYBC4HBP+7p7+/MB4JecGordb2Zjg2ONBQ50VZS7L3b3cncvLyrq8t58IiIiIhkllYFuNTDVzCaZWTaJSQ5LO7VZCtwZLH8QeNbdPVg/P5gFOwmYCrxiZnlmlg9gZnnAu4GNXRzrTuBXKTovERERkbSSsiFXd281s3uB3wFR4FF332RmXwcq3H0p8AjwQzOrBI6QCH0E7Z4ENgOtwD3u3mZmo4FfJuZNEAN+7O5PB2/5T8CTZnYXsBP4cKrOrbeqaxt4ZtM+PjVzUtiliIiISD9miQ6xgam8vNxT+dNf/7l8G9/4zev86p6ZXF4yNGXvIyIiIv2fma3pdMu2k/RLESl0xzWlFOTGWLRsa9iliIiISD+mQJdCQ3JifPK6iTy9aR9bD9aHXY6IiIj0Uwp0KfapmRPJjkb4z+Xbwi5FRERE+ikFuhQbOSSHD5WP5xdrq9l/tDHsckRERKQfUqDrAwtmldEaj/PoC9vDLkVERET6IQW6PlA6YjDvvWwcj6+qoq6hJexyREREpJ9RoOsjn5s9mfqmVn708s6wSxEREZF+RoGuj1xSXMjsC4r43sodNLa0hV2OiIiI9CMKdH1o4ZzJHKpv4udrd4ddioiIiPQjCnR96LrJI7i8ZCiLl2+jLT5wf6FDREREzi8Fuj5kZtw9ZzI7D5/gtxv3hl2OiIiI9BMKdH3sXdPGMHlkHouWbWUg/46uiIiInD8KdH0sGjE+N2cyG6uP8kLlobDLERERkX5AgS4Et19RzOiCHBYt2xp2KSIiItIPKNCFICcW5TMzJ7Gy8jAbdteGXY6IiIhkOAW6kHz0mlLyc2PqpRMREZG3TYEuJPm5WXzi2gn8duM+th86HnY5IiIiksEU6EL06ZmTyIpGWLxcvXQiIiJy7hToQlSUn8OHrhrPz9dUc+BoY9jliIiISIZSoAvZgtmTaY3HeXTljrBLERERkQylQBeyCSPyeM+lY3n85Z0cbWwJuxwRERHJQAp0aWDhnDKONbXy+MtVYZciIiIiGUiBLg1cUlzIrKkjeeSF7TS2tIVdjoiIiGQYBbo0cfecMg7VN/GLtdVhlyIiIiIZRoEuTVxXNoLLxheyePlW2uIedjkiIiKSQVIa6MxsrpltMbNKM7uvi+05ZvZEsH2VmU1M2nZ/sH6Lmd3Sab+oma0zs6eS1n3fzLab2frgMT2V53a+mRl3zyljx+ETPL1xX9jliIiISAZJWaAzsyjwbeBWYBpwh5lN69TsLqDG3acADwIPBPtOA+YDFwNzge8Ex2v3eeD1Lt72r9x9evBYf15PqA+8++IxTBqZx6JlW3FXL52IiIj0Tip76GYAle6+zd2bgSXAvE5t5gGPBcs/A242MwvWL3H3JnffDlQGx8PMxgPvBR5OYe2hiEaMBbMn81p1HSsrD4ddjoiIiGSIVAa6YmBX0uvdwbou27h7K1AHjDjDvt8C/hqId/Ge3zCzDWb2oJnlvO0zCMEHrixmVH4Oi5bp58BERESkdzJqUoSZvQ844O5ruth8P3ARcDUwHPhKN8dYYGYVZlZx8ODB1BV7jnJiUT5zwyReqDzEa7vrwi5HREREMkAqA101UJL0enywrss2ZhYDCoHDPew7E7jNzHaQGMK9ycx+BODuez2hCfgewRBtZ+6+2N3L3b28qKjo7Z1hinz0mlLyc2LqpRMREZFeSWWgWw1MNbNJZpZNYpLD0k5tlgJ3BssfBJ71xGyApcD8YBbsJGAq8Iq73+/u4919YnC8Z9394wBmNjZ4NuB2YGMKzy2lCnKz+Ph1E/jtxr3sOHQ87HJEREQkzaUs0AXXxN0L/I7EjNQn3X2TmX3dzG4Lmj0CjDCzSuCLwH3BvpuAJ4HNwNPAPe5+pp9QeNzMXgNeA0YC/3C+z6kvfXrmRGLRCItXbAu7FBEREUlzNpBvj1FeXu4VFRVhl9Gtv/nla/xszW5e+Mo7GZWfG3Y5IiIiEiIzW+Pu5V1ty6hJEQPNglmTaW2L872VO8IuRURERNKYAl0amzgyj1svGcuPXtrJ0caWsMsRERGRNKVAl+YWzinjWFMrP15VFXYpIiIikqYU6NLcpeMLuWHKSB55YTuNLWeaFyIiIiIDkQJdBrj7xjIOHmvil+s638ZPRERERIEuI1xfNoJLiwtZvHwbbfGBOytZREREuqZAlwHMjIVzyth+6Di/37Qv7HJEREQkzSjQZYi5l4xh4ojBPLRsKwP53oEiIiJyOgW6DBGNGAtml7Fhdx0vbT0cdjkiIiKSRhToMsgHrixm5JAcHlq2NexSREREJI0o0GWQ3Kwod90wiRVvHWJjdV3Y5YiIiEiaUKDLMB+7tpT8nBiL1EsnIiIiAQW6DFOQm8XHrp3Ab17by87Dx8MuR0RERNKAAl0G+szMicQiERYv3xZ2KSIiIpIGFOgy0KiCXP7sqmJ+umY3B481hV2OiIiIhEyBLkMtmF1GS1uc763cHnYpIiIiEjIFugw1aWQet14yhh++vJNjjS1hlyMiIiIhUqDLYAvnlHGssZUfr6oKuxQREREJkQJdBrts/FBmThnBIy9sp6m1LexyREREJCQKdBlu4ZwyDhxr4r/WVYddioiIiIREgS7D3TBlJJcUF/DdZdtoi3vY5YiIiEgIFOgynJmxcE4Z2w4d55nN+8IuR0REREKgQNcP3HrJWCaMGMxDy7bhrl46ERGRgUaBrh+IRowFsyfz6q5aXtp2OOxyREREpI8p0PUTf3bleEYOyWHRMv0cmIiIyECT0kBnZnPNbIuZVZrZfV1szzGzJ4Ltq8xsYtK2+4P1W8zslk77Rc1snZk9lbRuUnCMyuCY2ak8t3STmxXl0zMnsvzNg2ysrgu7HBEREelDKQt0ZhYFvg3cCkwD7jCzaZ2a3QXUuPsU4EHggWDfacB84GJgLvCd4HjtPg+83ulYDwAPBseqCY49oHz82gkMyYnx3eXqpRMRERlIUtlDNwOodPdt7t4MLAHmdWozD3gsWP4ZcLOZWbB+ibs3uft2oDI4HmY2Hngv8HD7QYJ9bgqOQXDM21NyVmmscFAWH7umlF9v2EPV4RNhlyMiIiJ9JJWBrhjYlfR6d7Cuyzbu3grUASPOsO+3gL8G4knbRwC1wTG6e68B4TM3TCIWibB4xdawSxEREZE+klGTIszsfcABd1/zNo6xwMwqzKzi4MGD57G69DC6IJcPXFnMTyt2c6i+KexyREREpA+kMtBVAyVJr8cH67psY2YxoBA43MO+M4HbzGwHiSHcm8zsR8E+Q4NjdPdeALj7Yncvd/fyoqKicz+7NLZg9mSa2+J8f+WOsEsRERGRPpDKQLcamBrMPs0mMclhaac2S4E7g+UPAs964s64S4H5wSzYScBU4BV3v9/dx7v7xOB4z7r7x4N9nguOQXDMX6Xw3NLa5KIhzL14DD94aQf1Ta1nbC8iIiKZLWWBLrie7V7gdyRmpD7p7pvM7OtmdlvQ7BFghJlVAl8E7gv23QQ8CWwGngbucfe2M7zlV4AvBscaERx7wFo4p4yjja38ZFVV2KWIiIhIitlA/qmo8vJyr6ioCLuMlLlj8ctsO1TP8r9+Jzmx6Jl3EBERkbRlZmvcvbyrbRk1KULOzt03lrH/aBO/Wrcn7FJEREQkhRTo+rFZU0dy8bgCFi3fSjw+cHtiRURE+jsFun7MzFg4p4xtB4/z+837wy5HREREUkSBrp+79ZIxlA4fzKJlWxnI10uKiIj0Zwp0/VwsGuHPZ09m/a5aVm0/EnY5IiIikgIKdAPAh64az8gh2Tz0vH4OTEREpD9SoBsAcrOifHrmJJa9eZDNe46GXY6IiIicZwp0A8THr5lAXnaURcvUSyciItLfKNANEIWDs/jYtRN4asMedh05EXY5IiIich4p0A0gn5k5iWjE+M8V28IuRURERM4jBboBZExhLh+4YjxPrN7FofqmsMsRERGR80SBboBZMGcyzW1xHntxR9iliIiIyHmiQDfAlBUN4d3TRvODl3ZS39QadjkiIiJyHijQDUAL55RR19DCkleqwi5FREREzgMFugHoitJhXDt5OA+v2E5zazzsckRERORtUqAboBbOKWPf0Ub+a3112KWIiIjI26RAN0DNuaCId4wt4LvLthKPe9jliIiIyNugQDdAmRkL50xm68Hj/OH1/WGXIyIiIm+DAt0A9t5Lx1IyfBAPLduKu3rpREREMpUC3QAWi0ZYMGsy66pqeWX7kbDLERERkXOkQDfAfai8hBF52SxatjXsUkREROQcKdANcLlZUT49cyLPbTnI63uPhl2OiIiInAMFOuET104kLzvKd9VLJyIikpEU6ITCwVncMaOU/96wl11HToRdjoiIiJwlBToB4K5Zk4gYPLxiW9iliIiIyFlKaaAzs7lmtsXMKs3svi6255jZE8H2VWY2MWnb/cH6LWZ2S7Au18xeMbNXzWyTmX0tqf33zWy7ma0PHtNTeW79zdjCQdw+vZgnKnZxuL4p7HJERETkLKQs0JlZFPg2cCswDbjDzKZ1anYXUOPuU4AHgQeCfacB84GLgbnAd4LjNQE3ufvlwHRgrpldm3S8v3L36cFjfarOrb/63JzJNLXGeezFHWGXIiIiImchlT10M4BKd9/m7s3AEmBepzbzgMeC5Z8BN5uZBeuXuHuTu28HKoEZnlAftM8KHroj7nkyZVQ+73rHaB57aSfHm1rDLkdERER6KZWBrhjYlfR6d7Cuyzbu3grUASN62tfMoma2HjgAPOPuq5LafcPMNpjZg2aW01VRZrbAzCrMrOLgwYPnfnb91MIby6hraGHJ6l1nbiwiIiJpIeMmRbh7m7tPB8YDM8zskmDT/cBFwNXAcOAr3ey/2N3L3b28qKioT2rOJFeWDuOaScN5eMU2mlvjYZcjIiIivZDKQFcNlCS9Hh+s67KNmcWAQuBwb/Z191rgORLX2OHue4Mh2SbgeySGfOUcLLyxjL11jSx9dU/YpYiIiEgvpDLQrQammtkkM8smMclhaac2S4E7g+UPAs964lfilwLzg1mwk4CpwCtmVmRmQwHMbBDwLuCN4PXY4NmA24GNKTy3fu3GC4q4aEw+i5ZtJR7XJYoiIiLpLmWBLrgm7l7gd8DrwJPuvsnMvm5mtwXNHgFGmFkl8EXgvmDfTcCTwGbgaeAed28DxgLPmdkGEoHxGXd/KjjW42b2GvAaMBL4h1SdW39nZtx9YxmVB+r54xsHwi5HREREzsASHWIDU3l5uVdUVIRdRlpqbYtz4788z6j8HH5+9/UkOj5FREQkLGa2xt3Lu9qWcZMipG/EohEWzJ7M2qpaVu+oCbscERER6YECnXTrQ1eVMDwvm0XLtoZdioiIiPRAgU66NSg7yqeun8izbxzgjX1Hwy5HREREuqFAJz365HUTGJwd5bvLtoVdioiIiHRDgU56NHRwNnfMKGXpq3vYXXMi7HJERESkCwp0ckafnTWJiMHDK7aHXYqIiIh0QYFOzmhs4SDmTS9myeoqjhxvDrscERER6USBTnpl4ZzJNLbEeezFHWGXIiIiIp0o0EmvTBmVz7umjeaxl3Zwork17HJEREQkiQKd9NrCOWXUnmhhySu7wi5FREREkijQSa9dNWEYMyYO5+EV22hpi4ddjoiIiAQU6OSs3H1jGXvqGlm6fk/YpYiIiEigV4HOzPLMLBIsX2Bmt5lZVmpLk3R044VFXDQmn+8u30o87mGXIyIiIvS+h245kGtmxcDvgU8A309VUZK+zIyFc8p4c389z75xIOxyREREhN4HOnP3E8AHgO+4+4eAi1NXlqSz9102luKhg1i0bGvYpYiIiAhnEejM7DrgY8Cvg3XR1JQk6S4WjfDnsyZRsbOG1TuOhF2OiIjIgNfbQPcF4H7gl+6+ycwmA8+lrixJdx+5upThedksel69dCIiImHrVaBz92Xufpu7PxBMjjjk7n+R4tokjQ3KjnLndRP54xsH2LLvWNjliIiIDGi9neX6YzMrMLM8YCOw2cz+KrWlSbr75HUTGJQV5bvL1UsnIiISpt4OuU5z96PA7cBvgUkkZrrKADYsL5s7ZpSydP0eqmsbwi5HRERkwOptoMsK7jt3O7DU3VsA3YRM+OysSQA8vGJbyJWIiIgMXL0NdN8FdgB5wHIzmwAcTVVRkjnGDR3EvOnFLHllFzXHm8MuR0REZEDq7aSIf3f3Ynd/jyfsBN6Z4tokQyycM5mGljYee2lH2KWIiIgMSL2dFFFoZt80s4rg8a8keutEmDo6nz95xygee3EHJ5pbwy5HRERkwOntkOujwDHgw8HjKPC9M+1kZnPNbIuZVZrZfV1szzGzJ4Ltq8xsYtK2+4P1W8zslmBdrpm9YmavmtkmM/taUvtJwTEqg2Nm9/Lc5Dy4+8Yyak608MTqXWGXIiIiMuD0NtCVufvfufu24PE1YHJPO5hZFPg2cCswDbjDzKZ1anYXUOPuU4AHgQeCfacB80n8vNhc4DvB8ZqAm9z9cmA6MNfMrg2O9QDwYHCsmuDY0keumjCcqycO4+EV22lpi4ddjoiIyIDS20DXYGY3tL8ws5nAme5TMQOoDAJgM7AEmNepzTzgsWD5Z8DNZmbB+iXu3uTu24FKYEZw/V590D4reHiwz03BMQiOeXsvz03Ok4VzyqiubeCpDXvCLkVERGRA6W2gWwh828x2mNkO4D+Az51hn2Igefxtd7Cuyzbu3grUASN62tfMoma2HjgAPOPuq4J9aoNjdPdekmLvvHAUF47OZ9Hz23DXXW1ERET6Sm9nub4aDHNeBlzm7leQ6BHrc+7e5u7TgfHADDO75Gz2N7MF7ZM7Dh48mJoiB6hIxPjcnMls2X+M57YcCLscERGRAaO3PXQAuPvR4BcjAL54hubVQEnS6/HBui7bmFkMKAQO92Zfd68FniNxjd1hYGhwjO7eq32/xe5e7u7lRUVFZzgFOVt/evk4iocO4qHn9XNgIiIifeWsAl0ndobtq4GpwezTbBKTHJZ2arMUuDNY/iDwrCfG6pYC84NZsJOAqcArZlZkZkMBzGwQ8C7gjWCf54JjEBzzV2/j3OQcZUUjfHbWJFbvqGHNziNhlyMiIjIgvJ1A1+NFUsH1bPcCvwNeB550901m9nUzuy1o9ggwwswqSfT43Rfsuwl4EtgMPA3c4+5twFjgOTPbQCIwPuPuTwXH+grwxeBYI4JjSwg+cnUJwwZn8dDz+jkwERGRvmA9XbxuZsfoOrgZMMjdY11syxjl5eVeUVERdhn90rf+8Cbf+sNb/P4vZ3PB6PywyxEREcl4ZrbG3cu72tZjD52757t7QReP/EwPc5Jad143kUFZUb67TL10IiIiqfZ2hlxFujUsL5uPXF3Cr9ZXs6f2TLcsFBERkbdDgU5S5rOzJgHw8IrtIVciIiLSvynQScqMHzaY2y4fx5LVVdQcbw67HBERkX5LgU5S6nNzyjjR3MYPXtoZdikiIiL9lgKdpNSFY/K5+aJRPPbSDhqa28IuR0REpF9SoJOUW3hjGUeON/Nkxa4zNxYREZGzpkAnKXf1xOGUTxjG4uXbaGmLh12OiIhIv6NAJ31i4Zwyqmsb+PWGvWGXIiIi0u8o0EmfuOmiUUwdNYRFy7bS06+TiIiIyNlToJM+EYkYC+eU8ca+Yzy/5WDY5YiIiPQrCnTSZ26bPo5xhbk8tGxr2KWIiIj0Kwp00meyohE+O2syr2w/wpqdNWGXIyIi0m8o0Emfmj+jhKGDs1ikXjoREZHzRoFO+tTg7BifvG4iz2zeT+WBY2GXIyIi0i8o0Emf+9T1E8nNirBo2bawSxEREekXFOikzw3Py2b+1aX8an01e2obwi5HREQk4ynQSSjuumEScYdHXtgedikiIiIZT4FOQlEyfDC3XT6On7xSRe2J5rDLERERyWgKdBKaz82ZzInmNn740s6wSxEREcloCnQSmovGFHDTRaP43os7aGhuC7scERGRjKVAJ6FaOKeMI8eb+emaXWGXIiIikrEU6CRUV08cxpWlQ1m8fButbfGwyxEREclICnQSKjPj7hunsLumgV+/tjfsckRERDKSAp2E7mPMUkIAACAASURBVOaLRjF11BAeen4r7h52OSIiIhknpYHOzOaa2RYzqzSz+7rYnmNmTwTbV5nZxKRt9wfrt5jZLcG6EjN7zsw2m9kmM/t8Uvuvmlm1ma0PHu9J5bnJ+ROJGAtmT+aNfcd4/s2DYZcjIiKScVIW6MwsCnwbuBWYBtxhZtM6NbsLqHH3KcCDwAPBvtOA+cDFwFzgO8HxWoEvufs04Frgnk7HfNDdpweP36Tq3OT8mze9mLGFuSx6fmvYpYiIiGScVPbQzQAq3X2buzcDS4B5ndrMAx4Lln8G3GxmFqxf4u5N7r4dqARmuPted18L4O7HgNeB4hSeg/SR7FiEu26YxKrtR1hbVRN2OSIiIhkllYGuGEi+F8VuTg9fJ9u4eytQB4zozb7B8OwVwKqk1fea2QYze9TMhr39U5C+dMeMUgoHZamXTkRE5Cxl5KQIMxsC/Bz4grsfDVY/BJQB04G9wL92s+8CM6sws4qDB3W9VjrJy4lx53UTeOb1/VQeqA+7HBERkYyRykBXDZQkvR4frOuyjZnFgELgcE/7mlkWiTD3uLv/or2Bu+939zZ3jwP/SWLI9zTuvtjdy929vKio6G2cnqTCnddPJCcWYfFy9dKJiIj0VioD3WpgqplNMrNsEpMclnZqsxS4M1j+IPCsJ+5bsRSYH8yCnQRMBV4Jrq97BHjd3b+ZfCAzG5v08v3AxvN+RpJyI4bk8JHyEn65rpq9dQ1hlyMiIpIRUhbogmvi7gV+R2LywpPuvsnMvm5mtwXNHgFGmFkl8EXgvmDfTcCTwGbgaeAed28DZgKfAG7q4vYk/2xmr5nZBuCdwF+m6twktT47azJxh0df2B52KSIiIhnBBvKNXMvLy72ioiLsMqQLn1+yjj9s3s+L991M4eCssMsREREJnZmtcffyrrZl5KQI6f8WzinjeHMbP3x5R9iliIiIpD0FOklL7xhbwI0XFvG9lTtobGkLuxwREZG0pkAnaevuOWUcPt7MTyt2nbmxiIjIAKZAJ2lrxqThXFE6lMUrttHaFg+7HBERkbSlQCdpy8xYOKeMXUca+M3GfWGXIyIikrYU6CStvesdoykryuOh57cykGdki4iI9ESBTtJaJGJ8bk4Zr+89yvK3DoVdjoiISFpSoJO0d/v0YsYU5PLQ85VhlyIiIpKWFOgk7WXHInx21iRe3naE9btqwy5HREQk7SjQSUaYP6OUgtwYi57fGnYpIiIiaUeBTjLCkJwYd14/kd9t3sfWg/VhlyMiIpJWFOgkY9x5/USyoxEWL9sWdikiIiJpRYFOMsbIITl8uLyEX6zbzb66xrDLERERSRsKdJJRFsyeTNzh0ZXbwy5FREQkbSjQSUYpGT6Y9146lsdf3kndiZawyxEREUkLCnSScRbOKeN4cxs/WrUz7FJERETSggKdZJxp4wqYc0ER31u5ncaWtrDLERERCZ0CnWSkhXPKOFTfzM/W7A67FBERkdAp0ElGunbycKaXDGXx8m20tsXDLkdERCRUCnSSkcyMhXPKqDpygt9u3Bd2OSIiIqFSoJOM9e5po5lclMeiZVtx97DLERERCY0CnWSsSMRYOLuMTXuOsuKtQ2GXIyIiEhoFOslo864Yx+iCHBYt2xp2KSIiIqFRoJOMlhOL8tkbJvPi1sO8uqs27HJERERCoUAnGe+Oa0opyI2pl05ERAaslAY6M5trZlvMrNLM7utie46ZPRFsX2VmE5O23R+s32JmtwTrSszsOTPbbGabzOzzSe2Hm9kzZvZW8Dwslecm6WNIToxPXDeBpzftY9vB+rDLERER6XMpC3RmFgW+DdwKTAPuMLNpnZrdBdS4+xTgQeCBYN9pwHzgYmAu8J3geK3Al9x9GnAtcE/SMe8D/ujuU4E/Bq9lgPjU9ZPIjkZYvHxb2KWIiIj0uVT20M0AKt19m7s3A0uAeZ3azAMeC5Z/BtxsZhasX+LuTe6+HagEZrj7XndfC+Dux4DXgeIujvUYcHuKzkvSUFF+Dh8qH88v1lZz4Ghj2OWIiIj0qVQGumJgV9Lr3ZwKX6e1cfdWoA4Y0Zt9g+HZK4BVwarR7r43WN4HjH67JyCZZcGsMlrjcR5ZuT3sUkRERPpURk6KMLMhwM+BL7j70c7bPXGX2S7vNGtmC8yswswqDh48mOJKpS+VjhjMey8bx+MvV1HX0BJ2OSIiIn0mlYGuGihJej0+WNdlGzOLAYXA4Z72NbMsEmHucXf/RVKb/WY2NmgzFjjQVVHuvtjdy929vKio6BxPTdLV52ZPpr6plcdX7Qy7FBERkT6TykC3GphqZpPMLJvEJIelndosBe4Mlj8IPBv0ri0F5gezYCcBU4FXguvrHgFed/dv9nCsO4FfnfczkrR3SXEhsy8o4tEXdtDY0hZ2OSIiIn0iZYEuuCbuXuB3JCYvPOnum8zs62Z2W9DsEWCEmVUCXySYmerum4Angc3A08A97t4GzAQ+AdxkZuuDx3uCY/0T8C4zewv4k+C1DEAL50zmUH0TP1+7O+xSRERE+oQN5B81Ly8v94qKirDLkPPM3bn92yupbWjh2S/dSDRiYZckIiLytpnZGncv72pbRk6KEOmJmXH3jWXsPHyC327ce+YdREREMpwCnfRL75o2hskj81i0bCsDuRdaREQGBgU66ZeiEWPB7MlsrD7KysrDYZcjIiKSUgp00m+9/8piRuXn8NCyyrBLERERSSkFOum3cmJR7rphEisrD7Nhd23Y5YiIiKRMLOwC+rN/X/XvfPn3XyYWiaX8kRXJ6pP3OZtH4raB4froNaX8x3OVLFq2le987KqwyxFJmbZ4G01tTTS1NtHU1kRjayNNrcFzsP5sl5vbmhk5eCSlhaWUFpZSUlhCaWEphTmFafHft4icokCXQleNvYovX/9lWuOt5/xobG2kJd5yTvuGLWKR1IZG61270tIjLHm9llHPXsjo/LxTITjaMQR3DsXnuj1iEf1lN0DEPX4yQHUXjNqD1Vktn0MAO1//zWdFssiJ5ZAbyyUWiXHoxKHTjp2fnX8y3JUWdAx7pYWlFOcXkxPLOS/1iEjv6D50/fQ+dO5O3ONvK0ym9OGpf48wnc+A2GG7nefjnYftUYv2aYB19y57onqzfMaAdJbHa4mfn98MjkVi5EQTISonlnN2y+e6XxfLObEcItbxSpy2eBv7j++nqq6KXXW7qKqrSiwfPbV88MTpv4s9ZsiYRNArOBX0kpeL8opOey8R6VlP96FToOungW6g6xxo//fSV/mvdbv41b3XMjQv2iH4tbR17AHt3CPaeXtv2nS73c/tPXuzPe7x0D7vqEXfdkA0s16Frua25vNWc3tPVHuYOePyWYak3hw7J5pDNBI9L+f0dsTjTn1zK/WNrRxrbOVYYwv1Ta0MG5zN2KG5jMzLIdLNTbobWhrYfXR3l2GvfflEy4kO++REcxhfML7LsNfe4zcke0hfnLpIxugp0GnIVfolMyNqUaKRKDnk8BfvvJxfrqnhqfUN3HfrRWGXlxJxj9MWbztvAfG8bu+hfftlBe5ObiyXvOw8hkeHd9kTddYBrIflWKT//O+vtS0ehLBWjjW1nFpubDn13NR62vr65OXmVnr6931W1BhTmMvYwkGMK8xl7NDguXAQY4fmMq5wAlOGT+myt9bdOdJwpGPQq9tF1dHE8h+3/5E9x/ac9o+SYbnDTg3ndjG0Oy5/XL/6HkXeDv2XIAPChBF53HrpWB5/eSf/451lFORmhV3SeRexCJFohKxoFoMYFHY50kuNLW0desTal48m95R1CmtHO61vaGk74/tkxyIU5MbIz80iPzfGkJwYI0cOPvk6PzeLgmB9+7q8nCg1x1vYU9fAntpG9tY1sLe2kYqdNex/bS8tbR0T4KCsKGMLcxk7tFPwC8LflGGXMH3M9C7ra423sufYnm6HdldWraSmsabDPhGLMC5/XJe9fO3LwwcN1zWtMiBoyFVDrgPGxuo63vf/XuArcy/i7hvLwi5HMpy7c6K5rUMASwSylk69Y516y5qSe8daaW4781D54OzoydDVHsYKTgaxjuvbg9nJdUGbnNj5HdaNx51D9U3sqWtkb23Dyee9dY3sCYLfgWONxDv9FZOfG2Nc0Kt3Wm/f0EGMLcwlN6vrWuub67u9jq99ufOQ/OCswd1ex1daWMr4gvEMytI/gCQz6Bq6bijQDTyfeGQVb+w7xoq/fme3f2lI/9cW907hq5X6Tr1fp/WONbZyNHl9U+tpYaUzM7oMX4ngdSp0FbQHr5ysTuuzyMuJEotm5uSBlrY4+482JkJeEPZOhr8g9B0+fvo1kcPzshM9fYWDGDf01PO4IPCNLsglq4vPJO5xDh4/2G3Yq6qrYl/9vtP2KxpcdNrQbvLw7pghYzSBQ9KCAl03FOgGnhcrD/HRh1cxaWQeeTlRomZEI50fEaIG0UiE2GnbLLFP1DrsG4sYkfZnSzyfsU20vW2EaOTU+3Xdpps6Iok2XZ/HqfX9acip5eT1Yi0dQlZ9h16wzj1kHZePN595iDIWsQ7h6uRypzCW3BNW0Gn94KxotxMJJKGxpY19Sb16e+s69fbVNnC0seOsdTMYlZ/TIfCNLTwV+MYNHUTRkK4ncTS1NlF9rPq0od2qo4nXO+t2Ut9c32GfrEgWxQXF3U/gKCihMLcwpZ+TCCjQdUuBbuBxdx54egvbDtbTFnfa3GmLO61tp5ZPe3SxvjXuxN1pbYuf1uZMvTZhiBjEIhEikeDZIBaNdAh9PYfOU+s7tD1Dm8Sxuw6snUNn3P30i/abkoNbYrmp9cxDlLlZEYbktA89dgxlQ3JOLRd0GJbsuD4npvsJpov6ptYOw7pdDe92vo4wFjFGF+SeCnxDcxlXOKhD6Bs2OOu079jdqWuq63Zot6quiupj1afdGqkgp6DH27QUFxSTHc1O+Wcl/ZsCXTcU6CQV3DuFvrgTj3d8Pj0YnmrbdaCM96LN6aGzV8G0U61t8Xg34fX08+l8/K7anGyb9P49GZITSxqSPL0nrHMP2ZDc5CHNxHBmdkzDYwOJu1PX0HJy4kbnHr69dY3sq2s87XrFnFjkZMDr0NsXhL+xQ3O7nEDVFm9jX/2+Hod2D5041GEfw07em++04BcM7RYNLtI/IqRHCnTdUKAT6XuJewRCazxOPH7qmeB6s6iGKCUF4nHn8PHmIOA1dBn+9h89fRLHkJxYMHP31G1akq/nG1s4iEHZp1+Pe6LlRMd787X3+B09tdzQ2tBhn9xYLiUFJR1+hSP5Ni0lBSXkZeel8mOSNKdA1w0FOhERadfaFufAsaaOga/9di11jeypbeRQfdNp+w0bnNV1D18wtDu6IPe0XmN353DD4R5/gWNv/d7T7s03fNBwLhhxAbNKZzGrdBY3lN7AsEHDUvq5SPpQoOuGAp2IiJyNptY29tc1UR309CUP67Y/1zV0/Ek4Mxg5JKfDjZiLhw7qEP6K8nNO651uaWs5dW++pKC3Yf8GVu9ZTXNbM4Zx6ehLmVU6i9kTZjOrdBZj88f25UcifUiBrhsKdCIicr6daG7tcCPmPcnPwRBv55nW7ZM4Og7vti8ngt+IvOyT19g1tDTwSvUrrKhawfKdy3lx14scbzkOwJThU5hdOptZExIhb9LQSbo2r59QoOuGAp2IiPQ1d+doY2swpNtwevirS9y7r7nTjO7sWISxhblMHJHHFaVDuaJ0GNNLhlI4KIuWthbW71vP8p3LWVG1ghVVKzjScASAcfnjmD1h9smQN61omu6rl6EU6LqhQCciIunIPTGJ41QPX/ttWhp5a/8xtuw/hntiOHdK0RCuLB3GFaVDuXLCMKYUDQFzNh/czIqdK1hetZzlO5ez59geIHEd3g2lNzC7dDazJ8zmirFX6DdxM4QCXTcU6EREJBMda2xhw+461u6sYW1VDet21VJ7InHtXn5OjOlBD96VpUO5omQYBYNibK/dnujBC0Je5ZFKAPKy8ri+5PqT1+DNKJ6hn0NLUwp03VCgExGR/sDd2X7oOGurahMBr6qWLfuOnrwNS1lRHleWDuPKCYmevKmj8jlwfN/Ja/BWVK3gtf2v4TjZ0WyuHnd1Yph2wmyuL7megpyCcE9QAAW6binQiYhIf1Xf1MqGXbWs21V7sievJujFG5ITY3rJ0MQwbTBc63aclbtWsnxnYoh2zd41tMZbiViE6WOmd5hJW5RXFPLZDUyhBTozmwv8GxAFHnb3f+q0PQf4AXAVcBj4iLvvCLbdD9wFtAF/4e6/C9Y/CrwPOODulyQd66vAnwMHg1V/4+6/6ak+BToRERko3J0dh0+wrioR7tburOWNpF68yUV5XFEyjCsnJEJe8bAIq/esOtmL99Lul2hsbQTgopEXdZhJW1pYGuKZDRyhBDoziwJvAu8CdgOrgTvcfXNSm/8BXObuC81sPvB+d/+ImU0DfgLMAMYBfwAucPc2M5sN1AM/6CLQ1bv7v/S2RgU6EREZyI43tSauxQuGaddV1XD4eDMAedlRLi8ZGgzVDmXauDy21712coj2haoXqGuqA2BC4YREuAtC3oUjLtStUlKgp0CXymktM4BKd98WFLEEmAdsTmozD/hqsPwz4D8s8SdgHrDE3ZuA7WZWGRzvJXdfbmYTU1i3iIjIgJCXE+O6shFcVzYCSPTiVR05cTLgra2q4aFlW0/+BvOkkXlcUXoLH5k8n/8zO59m28FLu1eyvGo5z2x9hh9t+BEARYOLTg7Pzp4wm8tGX0Y0cvpPpMn5k8pAVwzsSnq9G7imuzbu3mpmdcCIYP3LnfYt7sV73mtmnwQqgC+5e03nBma2AFgAUFqqLmIREZF2ZsaEEXlMGJHH+68YDyRulLxhd93JgLf8zYP8Ym01AIOzo1w+/mquLn0XC24dSkH+IV479PLJYdqfv/5zAApyCphZMvNkyCsfV05OLCe08+yP+tONZx4C/h7w4Plfgc90buTui4HFkBhy7csCRUREMs3g7BjXTh7BtZNP9eLtrmkIrsOrYW1VLYuXb6M16MWbMKKMK0vL+fqMrzF2+HH2Nq5j5a4XWFG1gvv/eD8AubFcrh1/7ckevGvHX8uQ7CGhnWN/kMpAVw2UJL0eH6zrqs1uM4sBhSQmR/Rm3w7cfX/7spn9J/DUOVcuIiIiXTIzSoYPpmT4YOZNTwyeNTS38Vp1+7V4NbxQeYhfrkv8tT0oaySXjb+TT07+ApOubaXBNvPqgZdZvnM531jxDf5++d8TtShXjbvqZMC7ofQGhg8aHuZpZpxUToqIkZgUcTOJMLYa+Ki7b0pqcw9wadKkiA+4+4fN7GLgx5yaFPFHYKq7twX7TQSe6jQpYqy77w2W/xK4xt3n91SjJkWIiIicf+29eO23TFlXVcOmPUdP9uKVDh/MlaVDuWhcDLLfYkd9BS/ueoFV1atobktMyrhk1CUdZtKOyx8X5imlhTBvW/Ie4FskblvyqLt/w8y+DlS4+1IzywV+CFwBHAHmJ02i+J8khkxbgS+4+2+D9T8BbgRGAvuBv3P3R8zsh8B0EkOuO4DPtQe87ijQiYiI9I3GljY2Br14a3cmrsc7cKwJgNysCJcVD+WSkkHk5e3gUOurrNv3Eit3raS+uR6AsmFlHWbSlg0rG3AzaXVj4W4o0ImIiITD3dlT13jq58uqatm0p46WtkQuGT9sENNL8hkxbA/H2cRbda+wsuoFDjccBmDskLEdZtJePOpiIhYJ85RSToGuGwp0IiIi6aOxpY1Ne07NqF27s5Z9RxM3M86JRbi0OJ/xo2poy9rMnoZ1vLJnJbuP7gZgWO4wbii94WTIu3LslWRFs8I8nfNOga4bCnQiIiLpbU9tw6mAV1XDpuqjNLfFARhXmMuUcQ1kD97C4dZX2XRoFW8deROAwVmDub7k+pM9eNcUX8OgrEFhnsrbpkDXDQU6ERGRzNLU2samPUcTky121bJuZw176hK9eNmxCFPHtFJQWMkJ28jWugo2H3oNx8mKZHF18dUnr8GbWTKTwtzCkM/m7CjQdUOBTkREJPPtq2s8ecuUtVW1vFZdR3NrohdvVEErRUU7aIttZm/DOjYfXkdrvBXDuHzM5cwunZ0Ypp0wi1F5o0I+k54p0HVDgU5ERKT/aWptY/OeoyeHatdV1VJd2wBALNrMmKLdJ4dp36xZQ2NrYtuFIy48OUQ7e8JsJgydEOZpnEaBrhsKdCIiIgPD/qONJ3vw1u6s4bXqOppa4zgtDBmyi4KhlZxgI9uPreFYcx0AJQUlJ8PdrNJZXDTyolBvlaJA1w0FOhERkYGpuTXO63uPnuzBW1tVw+6aBpw4xHZROLSSeNbr7GlYS03TAQCKBhd1mEl7+ZjLiUX67ldUFei6oUAnIiIi7Q4cazw1TLuzlg3VtTS0tNFqe8jJe5OsQVs40voqhxp3AZCfnc/M0pnMKp3Fhy/+MFOGT0lpfT0Fur6LlSIiIiJpbFR+LrdcPIZbLh4DQEtbnDf2HgtumXI1a6tqaKppIIdDtGZtZlDOm6zZvZGnK59m/JCpKQ90PVGgExEREelCVjTCpeMLuXR8IXdePxGAg8eagmvxrmFdVQ0bdteR01LDYL8y1FoV6ERERER6qSg/h3dfPIZ3B714rW1x3th3jJJhg0OtS4FORERE5BzFohEuKQ7/BsX9+1dsRURERAYABToRERGRDKdAJyIiIpLhFOhEREREMpwCnYiIiEiGU6ATERERyXAKdCIiIiIZToFOREREJMMp0ImIiIhkOAU6ERERkQxn7h52DaExs4PAzhS/zUjgUIrfQ86evpf0o+8kPel7ST/6TtJTX3wvE9y9qKsNAzrQ9QUzq3D38rDrkI70vaQffSfpSd9L+tF3kp7C/l405CoiIiKS4RToRERERDKcAl3qLQ67AOmSvpf0o+8kPel7ST/6TtJTqN+LrqETERERyXDqoRMRERHJcAp0KWRmc81si5lVmtl9YdcjYGaPmtkBM9sYdi2SYGYlZvacmW02s01m9vmwaxrozCzXzF4xs1eD7+RrYdckCWYWNbN1ZvZU2LVIgpntMLPXzGy9mVWEVoeGXFPDzKLAm8C7gN3AauAOd98camEDnJnNBuqBH7j7JWHXI2BmY4Gx7r7WzPKBNcDt+m8lPGZmQJ6715tZFvAC8Hl3fznk0gY8M/siUA4UuPv7wq5HEoEOKHf3UO8NqB661JkBVLr7NndvBpYA80KuacBz9+XAkbDrkFPcfa+7rw2WjwGvA8XhVjWweUJ98DIreOhf/yEzs/HAe4GHw65F0o8CXeoUA7uSXu9Gf0mJ9MjMJgJXAKvCrUSCob31wAHgGXfXdxK+bwF/DcTDLkQ6cOD3ZrbGzBaEVYQCnYikBTMbAvwc+IK7Hw27noHO3dvcfTowHphhZrpEIURm9j7ggLuvCbsWOc0N7n4lcCtwT3BpT59ToEudaqAk6fX4YJ2IdBJcp/Vz4HF3/0XY9cgp7l4LPAfMDbuWAW4mcFtwvdYS4CYz+1G4JQmAu1cHzweAX5K45KrPKdClzmpgqplNMrNsYD6wNOSaRNJOcAH+I8Dr7v7NsOsRMLMiMxsaLA8iMbnrjXCrGtjc/X53H+/uE0n8ffKsu3885LIGPDPLCyZzYWZ5wLuBUO6ioECXIu7eCtwL/I7ERd5PuvumcKsSM/sJ8BJwoZntNrO7wq5JmAl8gkSPw/rg8Z6wixrgxgLPmdkGEv84fcbddZsMkdONBl4ws1eBV4Bfu/vTYRSi25aIiIiIZDj10ImIiIhkOAU6ERERkQynQCciIiKS4RToRERERDKcAp2IiIhIhlOgE5G0ZmZuZv+a9PrLZvbV83Ts75vZB8/Hsc7wPh8ys9fN7LlO6yeaWUPS7VrWm9knz+P73mhmut2IyAAQC7sAEZEzaAI+YGb/6O6Hwi6mnZnFgvtN9sZdwJ+7+wtdbNsa/MSWyP/f3t2E1lGFYRz/P5FqaaJVqhvd2EaDWjBRSRZaq4g7RaJQpIYqLsQK1i8QXYhCcWGpoggFP6qgmIWlUCooKIqGutA2lLbR4getIoWABSUaJcWax8WcC2OIMb1VmoHnB5fMfe/cM2dmEV7ecy5vRNtSoYuIhe448Arw8MwPZlbYJE2Wv9dLGpG0U9JhSc9IGpK0W9KYpO7aMDdKGpX0TemX2WpMv1nSHkkHJN1bG3eXpHeAg7PMZ20Z/wtJm0rsSWAV8JqkzfO9aUmTkp6X9KWkjySdV+J9kj4r89oh6ZwSv0jSh5L2S9pbu8cuSdslfSVpuHTmoDyTg2WcZ+c7r4hYmJLQRUQTbAGGJC09ge/0AuuBS6k6UfTYHgC2Ahtq511I1XvxJuAlSYupKmoTtvuBfuAeScvL+VcCD9ruqV9M0vnAJuAGoA/olzRoeyMwCgzZfnSWeXbPWHK9tsQ7gVHbK4ER4KkSfxN4zPblwFgtPgxssd0LXA2Ml/gVwEPAZcAK4BpJy4BbgZVlnKf/7WFGxMKWhC4iFjzbv1AlMg+cwNf22B63fQw4BHxQ4mNUSVzLNtvTtr8FDgOXUPVjvFPSPuBzYBlwcTl/t+3vZrleP/CJ7aNlKXYYWD2PeR6y3Vd77SrxaeDtcvwWsKoktGfbHinxN4DVpZfkBbZ3ANiesv17bb5HbE8D+8q9TwBTVFXD24DWuRHRUEnoIqIpXqCqnHXWYscp/8ckdQCn1z47Vjuerr2f5u/7h2f2PzQgYEMtyVpuu5UQ/nZSd9G+dvs01p/Dn0Br798AsB24GTglvScj4r+ThC4iGsH2T8A2qqSu5XvgqnJ8C7CojaHXSOooe85WAF8D7wP3SVoEIKlHUudcg1A15r5O0rmSTgPWUi2VtqsDaO0PvAP41PYE8HNtWXYdMGL7V+CIpMEy3zMkLfmngSV1AUttv0e1N7H3JOYZEQtAfuUaEU3yHHB/ytkNlAAAAMRJREFU7f2rwE5J+6mqTO1Uz36gSsbOAtbbnpK0lWppcm/5EcFRYHCuQWyPS3oc+Jiqwveu7Z3zuH53Wdpted32i1T3MiDpCeBH4Pby+V1Ue/2WUC0R313i64CXJW0E/gDWzHHNM6me2+Iy10fmMc+IWMBkt1vFj4iI/4ukSdtdp3oeEdEMWXKNiIiIaLhU6CIiIiIaLhW6iIiIiIZLQhcRERHRcEnoIiIiIhouCV1EREREwyWhi4iIiGi4JHQRERERDfcXav+qUdhfhDUAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "12PYfqX3QrE-"
      },
      "source": [
        "## **6. Get Predictions**\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### (1) Testing"
      ],
      "metadata": {
        "id": "D5zQrJXM8vEv"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ey9FMYJxQqGs"
      },
      "source": [
        "def generate_predictions(model, df, num_labels, device=\"cpu\", batch_size=32):\n",
        "  num_iter = math.ceil(df.shape[0]/batch_size)\n",
        "  \n",
        "  pred_probs = np.array([]).reshape(0, num_labels)\n",
        "  \n",
        "  model.to(device)\n",
        "  model.eval()\n",
        "  \n",
        "  for i in range(num_iter):\n",
        "    df_subset = df.iloc[i*batch_size:(i+1)*batch_size,:]\n",
        "    X = df_subset[\"features\"].values.tolist()\n",
        "    masks = df_subset[\"masks\"].values.tolist()\n",
        "    X = torch.tensor(X)\n",
        "    masks = torch.tensor(masks, dtype=torch.long)\n",
        "    X = X.to(device)\n",
        "    masks = masks.to(device)\n",
        "    with torch.no_grad():\n",
        "      logits = model(input_ids=X, attention_mask=masks)\n",
        "      logits = logits.sigmoid().detach().cpu().numpy()\n",
        "      pred_probs = np.vstack([pred_probs, logits])\n",
        "  \n",
        "  return pred_probs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wo0YyAH6XGTh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8890fb2a-84e9-4a6b-84fd-c8a06b396aae"
      },
      "source": [
        "num_labels = len(label_cols)\n",
        "pred_probs = generate_predictions(model, test, num_labels, device=\"cuda\", batch_size=32)\n",
        "pred_probs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[7.23557954e-04, 2.81989138e-04, 3.07715498e-04, ...,\n",
              "        6.77508933e-06, 1.31822156e-03, 7.61014991e-04],\n",
              "       [5.03615593e-04, 7.38265298e-05, 3.91963311e-03, ...,\n",
              "        1.99255592e-05, 1.46217272e-02, 1.29244691e-02],\n",
              "       [2.80912849e-04, 5.89488845e-05, 4.16535558e-03, ...,\n",
              "        5.71500823e-05, 1.03602104e-03, 2.39653490e-03],\n",
              "       ...,\n",
              "       [1.00547762e-03, 3.08516261e-04, 1.21603988e-03, ...,\n",
              "        2.56056774e-05, 8.03853571e-03, 3.89410486e-03],\n",
              "       [2.90283177e-04, 6.21911895e-05, 6.15084253e-04, ...,\n",
              "        2.75567327e-05, 9.36885700e-02, 2.05611419e-02],\n",
              "       [3.10399942e-03, 1.22190657e-04, 2.15629418e-03, ...,\n",
              "        1.11240217e-04, 1.39068320e-01, 2.95488890e-02]])"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uVAWKam2ado5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "0636f2d6-9d3f-42a7-e209-beca285f0bf7"
      },
      "source": [
        "for i in label_cols:\n",
        "  test[i] = pred_probs[:,label_cols.index(i)]\n",
        "\n",
        "test.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
            "  \n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          id                               name  \\\n",
              "90077  90078              hairrecipe蜂蜜保濕營養護髮精華素   \n",
              "26263  26264                         一匙靈制菌洗衣精瓶裝   \n",
              "25412  25413  日本no1ariel日本進口三合一3d洗衣膠囊洗衣球室內晾乾六入箱   \n",
              "37753  37754              cnplaboratory蜂膠能量彈潤乳霜   \n",
              "89670  89671                       炫豔護髮染髮乳自然褐白染   \n",
              "\n",
              "                                                features  \\\n",
              "90077  [19, 1141, 11569, 1958, 5263, 5304, 5398, 6027...   \n",
              "26263  [19, 59, 23183, 1565, 299, 2440, 21816, 1191, ...   \n",
              "25412  [4531, 4615, 41, 8816, 1725, 161, 10046, 86, 1...   \n",
              "37753  [19, 4447, 1116, 2453, 18960, 14707, 592, 5398...   \n",
              "89670  [19, 17141, 21748, 2267, 4532, 6342, 4532, 325...   \n",
              "\n",
              "                                                   masks       bb霜       cc霜  \\\n",
              "90077  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.000724  0.000282   \n",
              "26263  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.000504  0.000074   \n",
              "25412  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.000281  0.000059   \n",
              "37753  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.002596  0.000229   \n",
              "89670  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.002355  0.000100   \n",
              "\n",
              "            dd霜     rtd調酒      中式香腸       乳酸菌  ...       電磁爐     電視遊戲機  \\\n",
              "90077  0.000308  0.000287  0.000375  0.000020  ...  0.000002  0.000006   \n",
              "26263  0.003920  0.003116  0.010640  0.000549  ...  0.000043  0.000033   \n",
              "25412  0.004165  0.001291  0.002252  0.000193  ...  0.000079  0.000028   \n",
              "37753  0.003085  0.000308  0.000125  0.000020  ...  0.000006  0.000016   \n",
              "89670  0.000275  0.000443  0.000659  0.000018  ...  0.000007  0.000010   \n",
              "\n",
              "             電鍋       電風扇        面膜     面霜/乳霜       香氛機      香氛蠟燭        鮮乳  \\\n",
              "90077  0.000154  0.000078  0.003564  0.063152  0.002056  0.000007  0.001318   \n",
              "26263  0.000022  0.000007  0.001343  0.002180  0.000737  0.000020  0.014622   \n",
              "25412  0.000018  0.000137  0.000692  0.006967  0.001913  0.000057  0.001036   \n",
              "37753  0.000152  0.000124  0.002361  0.315388  0.001108  0.000037  0.000684   \n",
              "89670  0.000378  0.000127  0.001362  0.040070  0.003629  0.000009  0.002084   \n",
              "\n",
              "           麥片穀類  \n",
              "90077  0.000761  \n",
              "26263  0.012924  \n",
              "25412  0.002397  \n",
              "37753  0.000405  \n",
              "89670  0.001049  \n",
              "\n",
              "[5 rows x 220 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0d23f3c8-c20a-42fe-b434-6834597c4fe2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "      <th>features</th>\n",
              "      <th>masks</th>\n",
              "      <th>bb霜</th>\n",
              "      <th>cc霜</th>\n",
              "      <th>dd霜</th>\n",
              "      <th>rtd調酒</th>\n",
              "      <th>中式香腸</th>\n",
              "      <th>乳酸菌</th>\n",
              "      <th>...</th>\n",
              "      <th>電磁爐</th>\n",
              "      <th>電視遊戲機</th>\n",
              "      <th>電鍋</th>\n",
              "      <th>電風扇</th>\n",
              "      <th>面膜</th>\n",
              "      <th>面霜/乳霜</th>\n",
              "      <th>香氛機</th>\n",
              "      <th>香氛蠟燭</th>\n",
              "      <th>鮮乳</th>\n",
              "      <th>麥片穀類</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>90077</th>\n",
              "      <td>90078</td>\n",
              "      <td>hairrecipe蜂蜜保濕營養護髮精華素</td>\n",
              "      <td>[19, 1141, 11569, 1958, 5263, 5304, 5398, 6027...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.000724</td>\n",
              "      <td>0.000282</td>\n",
              "      <td>0.000308</td>\n",
              "      <td>0.000287</td>\n",
              "      <td>0.000375</td>\n",
              "      <td>0.000020</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000002</td>\n",
              "      <td>0.000006</td>\n",
              "      <td>0.000154</td>\n",
              "      <td>0.000078</td>\n",
              "      <td>0.003564</td>\n",
              "      <td>0.063152</td>\n",
              "      <td>0.002056</td>\n",
              "      <td>0.000007</td>\n",
              "      <td>0.001318</td>\n",
              "      <td>0.000761</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26263</th>\n",
              "      <td>26264</td>\n",
              "      <td>一匙靈制菌洗衣精瓶裝</td>\n",
              "      <td>[19, 59, 23183, 1565, 299, 2440, 21816, 1191, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.000504</td>\n",
              "      <td>0.000074</td>\n",
              "      <td>0.003920</td>\n",
              "      <td>0.003116</td>\n",
              "      <td>0.010640</td>\n",
              "      <td>0.000549</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000043</td>\n",
              "      <td>0.000033</td>\n",
              "      <td>0.000022</td>\n",
              "      <td>0.000007</td>\n",
              "      <td>0.001343</td>\n",
              "      <td>0.002180</td>\n",
              "      <td>0.000737</td>\n",
              "      <td>0.000020</td>\n",
              "      <td>0.014622</td>\n",
              "      <td>0.012924</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25412</th>\n",
              "      <td>25413</td>\n",
              "      <td>日本no1ariel日本進口三合一3d洗衣膠囊洗衣球室內晾乾六入箱</td>\n",
              "      <td>[4531, 4615, 41, 8816, 1725, 161, 10046, 86, 1...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.000281</td>\n",
              "      <td>0.000059</td>\n",
              "      <td>0.004165</td>\n",
              "      <td>0.001291</td>\n",
              "      <td>0.002252</td>\n",
              "      <td>0.000193</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000079</td>\n",
              "      <td>0.000028</td>\n",
              "      <td>0.000018</td>\n",
              "      <td>0.000137</td>\n",
              "      <td>0.000692</td>\n",
              "      <td>0.006967</td>\n",
              "      <td>0.001913</td>\n",
              "      <td>0.000057</td>\n",
              "      <td>0.001036</td>\n",
              "      <td>0.002397</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37753</th>\n",
              "      <td>37754</td>\n",
              "      <td>cnplaboratory蜂膠能量彈潤乳霜</td>\n",
              "      <td>[19, 4447, 1116, 2453, 18960, 14707, 592, 5398...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.002596</td>\n",
              "      <td>0.000229</td>\n",
              "      <td>0.003085</td>\n",
              "      <td>0.000308</td>\n",
              "      <td>0.000125</td>\n",
              "      <td>0.000020</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000006</td>\n",
              "      <td>0.000016</td>\n",
              "      <td>0.000152</td>\n",
              "      <td>0.000124</td>\n",
              "      <td>0.002361</td>\n",
              "      <td>0.315388</td>\n",
              "      <td>0.001108</td>\n",
              "      <td>0.000037</td>\n",
              "      <td>0.000684</td>\n",
              "      <td>0.000405</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89670</th>\n",
              "      <td>89671</td>\n",
              "      <td>炫豔護髮染髮乳自然褐白染</td>\n",
              "      <td>[19, 17141, 21748, 2267, 4532, 6342, 4532, 325...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.002355</td>\n",
              "      <td>0.000100</td>\n",
              "      <td>0.000275</td>\n",
              "      <td>0.000443</td>\n",
              "      <td>0.000659</td>\n",
              "      <td>0.000018</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000007</td>\n",
              "      <td>0.000010</td>\n",
              "      <td>0.000378</td>\n",
              "      <td>0.000127</td>\n",
              "      <td>0.001362</td>\n",
              "      <td>0.040070</td>\n",
              "      <td>0.003629</td>\n",
              "      <td>0.000009</td>\n",
              "      <td>0.002084</td>\n",
              "      <td>0.001049</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 220 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0d23f3c8-c20a-42fe-b434-6834597c4fe2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0d23f3c8-c20a-42fe-b434-6834597c4fe2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0d23f3c8-c20a-42fe-b434-6834597c4fe2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### (2) For single-label classification"
      ],
      "metadata": {
        "id": "QdNMYK9h87Hx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the maximum probability for each row\n",
        "test['product'] = test.iloc[:,4:].idxmax(axis = 1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UqRkcrccnf0B",
        "outputId": "e8207ea5-e3c2-42c9-f087-d6cd22e15166"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### (3) Save the predicting results"
      ],
      "metadata": {
        "id": "bW4uDWHv8hXg"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dpE1pjPIAJGg",
        "outputId": "a323d7c0-1f00-44fa-ac41-27a68227b1bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "test_to_csv = test.reset_index()\n",
        "test_to_csv.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   index     id                               name  \\\n",
              "0  90077  90078              hairrecipe蜂蜜保濕營養護髮精華素   \n",
              "1  26263  26264                         一匙靈制菌洗衣精瓶裝   \n",
              "2  25412  25413  日本no1ariel日本進口三合一3d洗衣膠囊洗衣球室內晾乾六入箱   \n",
              "3  37753  37754              cnplaboratory蜂膠能量彈潤乳霜   \n",
              "4  89670  89671                       炫豔護髮染髮乳自然褐白染   \n",
              "\n",
              "                                            features  \\\n",
              "0  [19, 1141, 11569, 1958, 5263, 5304, 5398, 6027...   \n",
              "1  [19, 59, 23183, 1565, 299, 2440, 21816, 1191, ...   \n",
              "2  [4531, 4615, 41, 8816, 1725, 161, 10046, 86, 1...   \n",
              "3  [19, 4447, 1116, 2453, 18960, 14707, 592, 5398...   \n",
              "4  [19, 17141, 21748, 2267, 4532, 6342, 4532, 325...   \n",
              "\n",
              "                                               masks       bb霜       cc霜  \\\n",
              "0  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.000724  0.000282   \n",
              "1  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.000504  0.000074   \n",
              "2  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.000281  0.000059   \n",
              "3  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.002596  0.000229   \n",
              "4  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  0.002355  0.000100   \n",
              "\n",
              "        dd霜     rtd調酒      中式香腸  ...     電視遊戲機        電鍋       電風扇        面膜  \\\n",
              "0  0.000308  0.000287  0.000375  ...  0.000006  0.000154  0.000078  0.003564   \n",
              "1  0.003920  0.003116  0.010640  ...  0.000033  0.000022  0.000007  0.001343   \n",
              "2  0.004165  0.001291  0.002252  ...  0.000028  0.000018  0.000137  0.000692   \n",
              "3  0.003085  0.000308  0.000125  ...  0.000016  0.000152  0.000124  0.002361   \n",
              "4  0.000275  0.000443  0.000659  ...  0.000010  0.000378  0.000127  0.001362   \n",
              "\n",
              "      面霜/乳霜       香氛機      香氛蠟燭        鮮乳      麥片穀類  product  \n",
              "0  0.063152  0.002056  0.000007  0.001318  0.000761      洗髮精  \n",
              "1  0.002180  0.000737  0.000020  0.014622  0.012924      洗衣精  \n",
              "2  0.006967  0.001913  0.000057  0.001036  0.002397      洗衣精  \n",
              "3  0.315388  0.001108  0.000037  0.000684  0.000405    面霜/乳霜  \n",
              "4  0.040070  0.003629  0.000009  0.002084  0.001049      洗髮精  \n",
              "\n",
              "[5 rows x 222 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-07b7ada7-9b35-4ec9-bfb2-1338b504719a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "      <th>features</th>\n",
              "      <th>masks</th>\n",
              "      <th>bb霜</th>\n",
              "      <th>cc霜</th>\n",
              "      <th>dd霜</th>\n",
              "      <th>rtd調酒</th>\n",
              "      <th>中式香腸</th>\n",
              "      <th>...</th>\n",
              "      <th>電視遊戲機</th>\n",
              "      <th>電鍋</th>\n",
              "      <th>電風扇</th>\n",
              "      <th>面膜</th>\n",
              "      <th>面霜/乳霜</th>\n",
              "      <th>香氛機</th>\n",
              "      <th>香氛蠟燭</th>\n",
              "      <th>鮮乳</th>\n",
              "      <th>麥片穀類</th>\n",
              "      <th>product</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>90077</td>\n",
              "      <td>90078</td>\n",
              "      <td>hairrecipe蜂蜜保濕營養護髮精華素</td>\n",
              "      <td>[19, 1141, 11569, 1958, 5263, 5304, 5398, 6027...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.000724</td>\n",
              "      <td>0.000282</td>\n",
              "      <td>0.000308</td>\n",
              "      <td>0.000287</td>\n",
              "      <td>0.000375</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000006</td>\n",
              "      <td>0.000154</td>\n",
              "      <td>0.000078</td>\n",
              "      <td>0.003564</td>\n",
              "      <td>0.063152</td>\n",
              "      <td>0.002056</td>\n",
              "      <td>0.000007</td>\n",
              "      <td>0.001318</td>\n",
              "      <td>0.000761</td>\n",
              "      <td>洗髮精</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>26263</td>\n",
              "      <td>26264</td>\n",
              "      <td>一匙靈制菌洗衣精瓶裝</td>\n",
              "      <td>[19, 59, 23183, 1565, 299, 2440, 21816, 1191, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.000504</td>\n",
              "      <td>0.000074</td>\n",
              "      <td>0.003920</td>\n",
              "      <td>0.003116</td>\n",
              "      <td>0.010640</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000033</td>\n",
              "      <td>0.000022</td>\n",
              "      <td>0.000007</td>\n",
              "      <td>0.001343</td>\n",
              "      <td>0.002180</td>\n",
              "      <td>0.000737</td>\n",
              "      <td>0.000020</td>\n",
              "      <td>0.014622</td>\n",
              "      <td>0.012924</td>\n",
              "      <td>洗衣精</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>25412</td>\n",
              "      <td>25413</td>\n",
              "      <td>日本no1ariel日本進口三合一3d洗衣膠囊洗衣球室內晾乾六入箱</td>\n",
              "      <td>[4531, 4615, 41, 8816, 1725, 161, 10046, 86, 1...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.000281</td>\n",
              "      <td>0.000059</td>\n",
              "      <td>0.004165</td>\n",
              "      <td>0.001291</td>\n",
              "      <td>0.002252</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000028</td>\n",
              "      <td>0.000018</td>\n",
              "      <td>0.000137</td>\n",
              "      <td>0.000692</td>\n",
              "      <td>0.006967</td>\n",
              "      <td>0.001913</td>\n",
              "      <td>0.000057</td>\n",
              "      <td>0.001036</td>\n",
              "      <td>0.002397</td>\n",
              "      <td>洗衣精</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>37753</td>\n",
              "      <td>37754</td>\n",
              "      <td>cnplaboratory蜂膠能量彈潤乳霜</td>\n",
              "      <td>[19, 4447, 1116, 2453, 18960, 14707, 592, 5398...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.002596</td>\n",
              "      <td>0.000229</td>\n",
              "      <td>0.003085</td>\n",
              "      <td>0.000308</td>\n",
              "      <td>0.000125</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000016</td>\n",
              "      <td>0.000152</td>\n",
              "      <td>0.000124</td>\n",
              "      <td>0.002361</td>\n",
              "      <td>0.315388</td>\n",
              "      <td>0.001108</td>\n",
              "      <td>0.000037</td>\n",
              "      <td>0.000684</td>\n",
              "      <td>0.000405</td>\n",
              "      <td>面霜/乳霜</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>89670</td>\n",
              "      <td>89671</td>\n",
              "      <td>炫豔護髮染髮乳自然褐白染</td>\n",
              "      <td>[19, 17141, 21748, 2267, 4532, 6342, 4532, 325...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "      <td>0.002355</td>\n",
              "      <td>0.000100</td>\n",
              "      <td>0.000275</td>\n",
              "      <td>0.000443</td>\n",
              "      <td>0.000659</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000010</td>\n",
              "      <td>0.000378</td>\n",
              "      <td>0.000127</td>\n",
              "      <td>0.001362</td>\n",
              "      <td>0.040070</td>\n",
              "      <td>0.003629</td>\n",
              "      <td>0.000009</td>\n",
              "      <td>0.002084</td>\n",
              "      <td>0.001049</td>\n",
              "      <td>洗髮精</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 222 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-07b7ada7-9b35-4ec9-bfb2-1338b504719a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-07b7ada7-9b35-4ec9-bfb2-1338b504719a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-07b7ada7-9b35-4ec9-bfb2-1338b504719a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mF95PeFM_rDh"
      },
      "source": [
        "cwd = os.getcwd()\n",
        "pred_save_path = output_model_file = os.path.join(cwd, \"/content/drive/MyDrive/碩一下/金融科技/麻布數據/output/product_1_epoch.csv\")\n",
        "test_to_csv.to_csv(pred_save_path, index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}